2024-12-02 17:15:30,719 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-02 17:15:30,737 main DEBUG Took 0.014720 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 17:15:30,737 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 17:15:30,738 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 17:15:30,739 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 17:15:30,740 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,749 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 17:15:30,770 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,772 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,773 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,773 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,774 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,774 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,776 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,776 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,777 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,778 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,780 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,780 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,781 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,782 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,782 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,783 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,784 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,785 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:15:30,786 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,786 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 17:15:30,788 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:15:30,790 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 17:15:30,792 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 17:15:30,793 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 17:15:30,794 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 17:15:30,795 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 17:15:30,811 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 17:15:30,814 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 17:15:30,818 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 17:15:30,818 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 17:15:30,819 main DEBUG createAppenders(={Console}) 2024-12-02 17:15:30,820 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-02 17:15:30,820 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-02 17:15:30,820 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-02 17:15:30,821 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 17:15:30,821 main DEBUG OutputStream closed 2024-12-02 17:15:30,821 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 17:15:30,822 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 17:15:30,822 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-02 17:15:30,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 17:15:30,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 17:15:30,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 17:15:30,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 17:15:30,938 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 17:15:30,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 17:15:30,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 17:15:30,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 17:15:30,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 17:15:30,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 17:15:30,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 17:15:30,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 17:15:30,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 17:15:30,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 17:15:30,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 17:15:30,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 17:15:30,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 17:15:30,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 17:15:30,949 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 17:15:30,949 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-02 17:15:30,949 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 17:15:30,951 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-02T17:15:30,975 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-02 17:15:30,978 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 17:15:30,979 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T17:15:31,225 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48 2024-12-02T17:15:31,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T17:15:31,255 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841, deleteOnExit=true 2024-12-02T17:15:31,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T17:15:31,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/test.cache.data in system properties and HBase conf 2024-12-02T17:15:31,257 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T17:15:31,257 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.log.dir in system properties and HBase conf 2024-12-02T17:15:31,258 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T17:15:31,258 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T17:15:31,259 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T17:15:31,360 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T17:15:31,471 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T17:15:31,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T17:15:31,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T17:15:31,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T17:15:31,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T17:15:31,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T17:15:31,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T17:15:31,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T17:15:31,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T17:15:31,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T17:15:31,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/nfs.dump.dir in system properties and HBase conf 2024-12-02T17:15:31,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/java.io.tmpdir in system properties and HBase conf 2024-12-02T17:15:31,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T17:15:31,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T17:15:31,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T17:15:33,234 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T17:15:33,433 INFO [Time-limited test {}] log.Log(170): Logging initialized @3517ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T17:15:33,588 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T17:15:33,672 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T17:15:33,735 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T17:15:33,736 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T17:15:33,740 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T17:15:33,777 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T17:15:33,791 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.log.dir/,AVAILABLE} 2024-12-02T17:15:33,792 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T17:15:34,276 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/java.io.tmpdir/jetty-localhost-39877-hadoop-hdfs-3_4_1-tests_jar-_-any-17211564102485110665/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T17:15:34,290 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:39877} 2024-12-02T17:15:34,291 INFO [Time-limited test {}] server.Server(415): Started @4382ms 2024-12-02T17:15:35,084 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T17:15:35,098 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T17:15:35,100 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T17:15:35,101 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T17:15:35,101 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T17:15:35,105 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.log.dir/,AVAILABLE} 2024-12-02T17:15:35,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T17:15:35,281 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/java.io.tmpdir/jetty-localhost-38997-hadoop-hdfs-3_4_1-tests_jar-_-any-13371223785219383158/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T17:15:35,282 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:38997} 2024-12-02T17:15:35,282 INFO [Time-limited test {}] server.Server(415): Started @5373ms 2024-12-02T17:15:35,383 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T17:15:35,695 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T17:15:35,708 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T17:15:35,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T17:15:35,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T17:15:35,779 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T17:15:35,783 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.log.dir/,AVAILABLE} 2024-12-02T17:15:35,784 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T17:15:35,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/java.io.tmpdir/jetty-localhost-40573-hadoop-hdfs-3_4_1-tests_jar-_-any-4004484757779580751/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T17:15:35,990 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:40573} 2024-12-02T17:15:35,990 INFO [Time-limited test {}] server.Server(415): Started @6081ms 2024-12-02T17:15:35,993 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T17:15:36,136 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T17:15:36,142 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T17:15:36,150 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T17:15:36,150 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T17:15:36,151 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T17:15:36,152 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.log.dir/,AVAILABLE} 2024-12-02T17:15:36,153 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T17:15:36,235 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data1/current/BP-1138772532-172.17.0.3-1733159732851/current, will proceed with Du for space computation calculation, 2024-12-02T17:15:36,236 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data2/current/BP-1138772532-172.17.0.3-1733159732851/current, will proceed with Du for space computation calculation, 2024-12-02T17:15:36,241 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data3/current/BP-1138772532-172.17.0.3-1733159732851/current, will proceed with Du for space computation calculation, 2024-12-02T17:15:36,242 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data4/current/BP-1138772532-172.17.0.3-1733159732851/current, will proceed with Du for space computation calculation, 2024-12-02T17:15:36,326 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/java.io.tmpdir/jetty-localhost-39135-hadoop-hdfs-3_4_1-tests_jar-_-any-4238538501486650593/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T17:15:36,328 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:39135} 2024-12-02T17:15:36,328 INFO [Time-limited test {}] server.Server(415): Started @6419ms 2024-12-02T17:15:36,331 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T17:15:36,342 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T17:15:36,342 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T17:15:36,447 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa3dc5e033b2fa154 with lease ID 0x4373559dc15ecd56: Processing first storage report for DS-dd8dc82b-5639-4510-9136-c7aeaf6c3351 from datanode DatanodeRegistration(127.0.0.1:40475, datanodeUuid=bc6d0b55-28fe-4b0b-b2ec-d095bd55a2af, infoPort=37663, infoSecurePort=0, ipcPort=36001, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851) 2024-12-02T17:15:36,449 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa3dc5e033b2fa154 with lease ID 0x4373559dc15ecd56: from storage DS-dd8dc82b-5639-4510-9136-c7aeaf6c3351 node DatanodeRegistration(127.0.0.1:40475, datanodeUuid=bc6d0b55-28fe-4b0b-b2ec-d095bd55a2af, infoPort=37663, infoSecurePort=0, ipcPort=36001, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T17:15:36,449 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa6a24040d877c1a3 with lease ID 0x4373559dc15ecd57: Processing first storage report for DS-a26ba80d-4012-4587-b072-51db620caa12 from datanode DatanodeRegistration(127.0.0.1:41165, datanodeUuid=d84305b4-2c40-456c-81a0-6f632adccdf2, infoPort=44061, infoSecurePort=0, ipcPort=38281, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851) 2024-12-02T17:15:36,450 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa6a24040d877c1a3 with lease ID 0x4373559dc15ecd57: from storage DS-a26ba80d-4012-4587-b072-51db620caa12 node DatanodeRegistration(127.0.0.1:41165, datanodeUuid=d84305b4-2c40-456c-81a0-6f632adccdf2, infoPort=44061, infoSecurePort=0, ipcPort=38281, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T17:15:36,450 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa3dc5e033b2fa154 with lease ID 0x4373559dc15ecd56: Processing first storage report for DS-cff49436-e297-4c3b-a206-1adf39e889e9 from datanode DatanodeRegistration(127.0.0.1:40475, datanodeUuid=bc6d0b55-28fe-4b0b-b2ec-d095bd55a2af, infoPort=37663, infoSecurePort=0, ipcPort=36001, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851) 2024-12-02T17:15:36,450 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa3dc5e033b2fa154 with lease ID 0x4373559dc15ecd56: from storage DS-cff49436-e297-4c3b-a206-1adf39e889e9 node DatanodeRegistration(127.0.0.1:40475, datanodeUuid=bc6d0b55-28fe-4b0b-b2ec-d095bd55a2af, infoPort=37663, infoSecurePort=0, ipcPort=36001, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T17:15:36,454 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa6a24040d877c1a3 with lease ID 0x4373559dc15ecd57: Processing first storage report for DS-869965a3-7b18-48c1-bc77-ea3e22bf6b2d from datanode DatanodeRegistration(127.0.0.1:41165, datanodeUuid=d84305b4-2c40-456c-81a0-6f632adccdf2, infoPort=44061, infoSecurePort=0, ipcPort=38281, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851) 2024-12-02T17:15:36,454 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa6a24040d877c1a3 with lease ID 0x4373559dc15ecd57: from storage DS-869965a3-7b18-48c1-bc77-ea3e22bf6b2d node DatanodeRegistration(127.0.0.1:41165, datanodeUuid=d84305b4-2c40-456c-81a0-6f632adccdf2, infoPort=44061, infoSecurePort=0, ipcPort=38281, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T17:15:36,572 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data5/current/BP-1138772532-172.17.0.3-1733159732851/current, will proceed with Du for space computation calculation, 2024-12-02T17:15:36,585 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data6/current/BP-1138772532-172.17.0.3-1733159732851/current, will proceed with Du for space computation calculation, 2024-12-02T17:15:36,689 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T17:15:36,699 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb3237bfe03d04380 with lease ID 0x4373559dc15ecd58: Processing first storage report for DS-668e2095-1fca-40fa-bb76-12a999cf7985 from datanode DatanodeRegistration(127.0.0.1:38149, datanodeUuid=b3255ec2-de5d-4782-8cae-b92c55db2949, infoPort=46411, infoSecurePort=0, ipcPort=39017, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851) 2024-12-02T17:15:36,699 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb3237bfe03d04380 with lease ID 0x4373559dc15ecd58: from storage DS-668e2095-1fca-40fa-bb76-12a999cf7985 node DatanodeRegistration(127.0.0.1:38149, datanodeUuid=b3255ec2-de5d-4782-8cae-b92c55db2949, infoPort=46411, infoSecurePort=0, ipcPort=39017, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T17:15:36,700 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb3237bfe03d04380 with lease ID 0x4373559dc15ecd58: Processing first storage report for DS-b3917946-8757-47fb-91a2-f6dda405eec5 from datanode DatanodeRegistration(127.0.0.1:38149, datanodeUuid=b3255ec2-de5d-4782-8cae-b92c55db2949, infoPort=46411, infoSecurePort=0, ipcPort=39017, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851) 2024-12-02T17:15:36,700 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb3237bfe03d04380 with lease ID 0x4373559dc15ecd58: from storage DS-b3917946-8757-47fb-91a2-f6dda405eec5 node DatanodeRegistration(127.0.0.1:38149, datanodeUuid=b3255ec2-de5d-4782-8cae-b92c55db2949, infoPort=46411, infoSecurePort=0, ipcPort=39017, storageInfo=lv=-57;cid=testClusterID;nsid=1040616293;c=1733159732851), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T17:15:36,979 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48 2024-12-02T17:15:37,149 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/zookeeper_0, clientPort=60644, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T17:15:37,162 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60644 2024-12-02T17:15:37,187 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:37,192 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:37,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741825_1001 (size=7) 2024-12-02T17:15:37,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741825_1001 (size=7) 2024-12-02T17:15:37,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741825_1001 (size=7) 2024-12-02T17:15:38,168 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 with version=8 2024-12-02T17:15:38,168 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/hbase-staging 2024-12-02T17:15:38,307 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T17:15:38,652 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3b3375d1a925:0 server-side Connection retries=45 2024-12-02T17:15:38,664 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:38,664 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:38,670 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T17:15:38,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:38,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T17:15:38,883 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T17:15:38,981 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T17:15:38,998 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T17:15:39,004 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T17:15:39,042 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8294 (auto-detected) 2024-12-02T17:15:39,044 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-02T17:15:39,095 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37351 2024-12-02T17:15:39,126 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37351 connecting to ZooKeeper ensemble=127.0.0.1:60644 2024-12-02T17:15:39,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373510x0, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T17:15:39,189 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37351-0x1009bfce1af0000 connected 2024-12-02T17:15:39,285 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:39,301 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:39,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:15:39,342 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567, hbase.cluster.distributed=false 2024-12-02T17:15:39,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T17:15:39,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37351 2024-12-02T17:15:39,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37351 2024-12-02T17:15:39,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37351 2024-12-02T17:15:39,394 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37351 2024-12-02T17:15:39,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37351 2024-12-02T17:15:39,572 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3b3375d1a925:0 server-side Connection retries=45 2024-12-02T17:15:39,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:39,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:39,576 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T17:15:39,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:39,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T17:15:39,580 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T17:15:39,584 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T17:15:39,597 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39557 2024-12-02T17:15:39,605 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39557 connecting to ZooKeeper ensemble=127.0.0.1:60644 2024-12-02T17:15:39,607 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:39,611 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:39,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:395570x0, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T17:15:39,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:395570x0, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:15:39,670 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T17:15:39,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39557-0x1009bfce1af0001 connected 2024-12-02T17:15:39,705 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T17:15:39,714 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T17:15:39,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T17:15:39,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39557 2024-12-02T17:15:39,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39557 2024-12-02T17:15:39,785 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39557 2024-12-02T17:15:39,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39557 2024-12-02T17:15:39,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39557 2024-12-02T17:15:39,839 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3b3375d1a925:0 server-side Connection retries=45 2024-12-02T17:15:39,841 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:39,842 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:39,842 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T17:15:39,843 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:39,843 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T17:15:39,845 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T17:15:39,847 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T17:15:39,866 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33117 2024-12-02T17:15:39,873 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33117 connecting to ZooKeeper ensemble=127.0.0.1:60644 2024-12-02T17:15:39,879 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:39,893 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:39,923 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:331170x0, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:15:39,924 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T17:15:39,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:331170x0, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T17:15:39,957 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T17:15:39,959 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:331170x0, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T17:15:39,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:331170x0, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T17:15:39,965 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33117-0x1009bfce1af0002 connected 2024-12-02T17:15:39,993 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33117 2024-12-02T17:15:40,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33117 2024-12-02T17:15:40,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33117 2024-12-02T17:15:40,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33117 2024-12-02T17:15:40,032 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33117 2024-12-02T17:15:40,079 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3b3375d1a925:0 server-side Connection retries=45 2024-12-02T17:15:40,079 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:40,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:40,080 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T17:15:40,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:15:40,081 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T17:15:40,081 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T17:15:40,081 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T17:15:40,086 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34079 2024-12-02T17:15:40,088 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34079 connecting to ZooKeeper ensemble=127.0.0.1:60644 2024-12-02T17:15:40,089 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:40,095 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:40,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340790x0, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T17:15:40,118 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:340790x0, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:15:40,119 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T17:15:40,121 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34079-0x1009bfce1af0003 connected 2024-12-02T17:15:40,138 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T17:15:40,140 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T17:15:40,144 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T17:15:40,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34079 2024-12-02T17:15:40,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34079 2024-12-02T17:15:40,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34079 2024-12-02T17:15:40,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34079 2024-12-02T17:15:40,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34079 2024-12-02T17:15:40,201 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3b3375d1a925:37351 2024-12-02T17:15:40,204 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3b3375d1a925,37351,1733159738385 2024-12-02T17:15:40,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:15:40,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:15:40,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:15:40,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:15:40,221 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3b3375d1a925,37351,1733159738385 2024-12-02T17:15:40,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T17:15:40,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T17:15:40,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:40,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:40,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T17:15:40,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:40,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:40,263 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T17:15:40,264 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3b3375d1a925,37351,1733159738385 from backup master directory 2024-12-02T17:15:40,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:15:40,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:15:40,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:15:40,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3b3375d1a925,37351,1733159738385 2024-12-02T17:15:40,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:15:40,269 WARN [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T17:15:40,269 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3b3375d1a925,37351,1733159738385 2024-12-02T17:15:40,273 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T17:15:40,281 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T17:15:40,392 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/hbase.id] with ID: d8980728-5223-41da-bed8-9a75b45a5f07 2024-12-02T17:15:40,392 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/.tmp/hbase.id 2024-12-02T17:15:40,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741826_1002 (size=42) 2024-12-02T17:15:40,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741826_1002 (size=42) 2024-12-02T17:15:40,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741826_1002 (size=42) 2024-12-02T17:15:40,465 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/.tmp/hbase.id]:[hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/hbase.id] 2024-12-02T17:15:40,523 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:15:40,530 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T17:15:40,555 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-12-02T17:15:40,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:40,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:40,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:40,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:40,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741827_1003 (size=196) 2024-12-02T17:15:40,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741827_1003 (size=196) 2024-12-02T17:15:40,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741827_1003 (size=196) 2024-12-02T17:15:40,619 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:15:40,622 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T17:15:40,643 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,648 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T17:15:40,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741828_1004 (size=1189) 2024-12-02T17:15:40,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741828_1004 (size=1189) 2024-12-02T17:15:40,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741828_1004 (size=1189) 2024-12-02T17:15:40,767 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store 2024-12-02T17:15:40,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741829_1005 (size=34) 2024-12-02T17:15:40,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741829_1005 (size=34) 2024-12-02T17:15:40,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741829_1005 (size=34) 2024-12-02T17:15:40,887 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T17:15:40,896 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:40,898 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T17:15:40,898 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:15:40,898 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:15:40,900 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-12-02T17:15:40,900 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:15:40,901 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:15:40,903 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733159740898Disabling compacts and flushes for region at 1733159740898Disabling writes for close at 1733159740900 (+2 ms)Writing region close event to WAL at 1733159740901 (+1 ms)Closed at 1733159740901 2024-12-02T17:15:40,906 WARN [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/.initializing 2024-12-02T17:15:40,906 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/WALs/3b3375d1a925,37351,1733159738385 2024-12-02T17:15:40,921 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T17:15:40,958 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b3375d1a925%2C37351%2C1733159738385, suffix=, logDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/WALs/3b3375d1a925,37351,1733159738385, archiveDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/oldWALs, maxLogs=10 2024-12-02T17:15:40,997 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/WALs/3b3375d1a925,37351,1733159738385/3b3375d1a925%2C37351%2C1733159738385.1733159740966, exclude list is [], retry=0 2024-12-02T17:15:41,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40475,DS-dd8dc82b-5639-4510-9136-c7aeaf6c3351,DISK] 2024-12-02T17:15:41,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41165,DS-a26ba80d-4012-4587-b072-51db620caa12,DISK] 2024-12-02T17:15:41,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38149,DS-668e2095-1fca-40fa-bb76-12a999cf7985,DISK] 2024-12-02T17:15:41,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T17:15:41,080 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/WALs/3b3375d1a925,37351,1733159738385/3b3375d1a925%2C37351%2C1733159738385.1733159740966 2024-12-02T17:15:41,081 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46411:46411),(127.0.0.1/127.0.0.1:44061:44061),(127.0.0.1/127.0.0.1:37663:37663)] 2024-12-02T17:15:41,082 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:15:41,082 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:41,086 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,087 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T17:15:41,167 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:41,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:41,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T17:15:41,176 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:41,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:41,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T17:15:41,180 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:41,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:41,182 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T17:15:41,185 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:41,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:41,187 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,191 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,192 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,199 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,200 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,203 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T17:15:41,216 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:15:41,243 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:41,252 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64891103, jitterRate=-0.03304721415042877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T17:15:41,262 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733159741103Initializing all the Stores at 1733159741105 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159741106 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159741107 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159741107Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159741107Cleaning up temporary data from old regions at 1733159741200 (+93 ms)Region opened successfully at 1733159741262 (+62 ms) 2024-12-02T17:15:41,269 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T17:15:41,352 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2408eae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3b3375d1a925/172.17.0.3:0 2024-12-02T17:15:41,402 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T17:15:41,419 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:41,419 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:41,423 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:41,425 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T17:15:41,434 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-02T17:15:41,434 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T17:15:41,477 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T17:15:41,489 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T17:15:41,496 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T17:15:41,499 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T17:15:41,503 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T17:15:41,504 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T17:15:41,508 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T17:15:41,529 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T17:15:41,536 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T17:15:41,543 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T17:15:41,545 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T17:15:41,573 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T17:15:41,575 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T17:15:41,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T17:15:41,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T17:15:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T17:15:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T17:15:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,589 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3b3375d1a925,37351,1733159738385, sessionid=0x1009bfce1af0000, setting cluster-up flag (Was=false) 2024-12-02T17:15:41,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,621 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T17:15:41,623 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3b3375d1a925,37351,1733159738385 2024-12-02T17:15:41,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:41,638 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T17:15:41,641 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3b3375d1a925,37351,1733159738385 2024-12-02T17:15:41,660 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T17:15:41,692 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(746): ClusterId : d8980728-5223-41da-bed8-9a75b45a5f07 2024-12-02T17:15:41,695 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(746): ClusterId : d8980728-5223-41da-bed8-9a75b45a5f07 2024-12-02T17:15:41,696 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T17:15:41,696 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T17:15:41,698 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(746): ClusterId : d8980728-5223-41da-bed8-9a75b45a5f07 2024-12-02T17:15:41,699 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T17:15:41,709 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T17:15:41,709 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T17:15:41,713 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T17:15:41,714 DEBUG [RS:1;3b3375d1a925:33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5523d416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3b3375d1a925/172.17.0.3:0 2024-12-02T17:15:41,721 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T17:15:41,721 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T17:15:41,737 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T17:15:41,737 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T17:15:41,745 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T17:15:41,746 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T17:15:41,746 DEBUG [RS:0;3b3375d1a925:39557 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f98052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3b3375d1a925/172.17.0.3:0 2024-12-02T17:15:41,747 DEBUG [RS:2;3b3375d1a925:34079 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cdb8991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3b3375d1a925/172.17.0.3:0 2024-12-02T17:15:41,767 DEBUG [RS:1;3b3375d1a925:33117 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3b3375d1a925:33117 2024-12-02T17:15:41,779 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T17:15:41,779 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T17:15:41,779 DEBUG [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T17:15:41,782 DEBUG [RS:2;3b3375d1a925:34079 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;3b3375d1a925:34079 2024-12-02T17:15:41,782 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T17:15:41,783 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T17:15:41,783 DEBUG [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T17:15:41,786 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(2659): reportForDuty to master=3b3375d1a925,37351,1733159738385 with port=33117, startcode=1733159739837 2024-12-02T17:15:41,790 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(2659): reportForDuty to master=3b3375d1a925,37351,1733159738385 with port=34079, startcode=1733159740078 2024-12-02T17:15:41,793 DEBUG [RS:0;3b3375d1a925:39557 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3b3375d1a925:39557 2024-12-02T17:15:41,793 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T17:15:41,793 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T17:15:41,794 DEBUG [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T17:15:41,796 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(2659): reportForDuty to master=3b3375d1a925,37351,1733159738385 with port=39557, startcode=1733159739502 2024-12-02T17:15:41,824 DEBUG [RS:0;3b3375d1a925:39557 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T17:15:41,826 DEBUG [RS:1;3b3375d1a925:33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T17:15:41,827 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:41,843 DEBUG [RS:2;3b3375d1a925:34079 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T17:15:41,856 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T17:15:41,872 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T17:15:41,885 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3b3375d1a925,37351,1733159738385 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T17:15:41,897 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3b3375d1a925:0, corePoolSize=5, maxPoolSize=5 2024-12-02T17:15:41,897 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3b3375d1a925:0, corePoolSize=5, maxPoolSize=5 2024-12-02T17:15:41,897 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3b3375d1a925:0, corePoolSize=5, maxPoolSize=5 2024-12-02T17:15:41,898 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3b3375d1a925:0, corePoolSize=5, maxPoolSize=5 2024-12-02T17:15:41,898 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3b3375d1a925:0, corePoolSize=10, maxPoolSize=10 2024-12-02T17:15:41,898 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:41,898 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3b3375d1a925:0, corePoolSize=2, maxPoolSize=2 2024-12-02T17:15:41,898 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:41,919 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58157, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T17:15:41,923 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54077, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T17:15:41,926 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46283, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T17:15:41,931 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:41,941 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:41,942 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159771942 2024-12-02T17:15:41,944 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T17:15:41,944 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:41,945 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T17:15:41,953 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T17:15:41,953 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T17:15:41,954 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T17:15:41,954 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T17:15:41,957 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:41,958 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T17:15:41,958 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:41,966 DEBUG [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-02T17:15:41,966 DEBUG [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-02T17:15:41,966 DEBUG [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-02T17:15:41,966 WARN [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-02T17:15:41,969 WARN [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-02T17:15:41,969 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T17:15:41,970 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:41,971 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T17:15:41,972 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T17:15:41,972 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T17:15:41,988 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T17:15:41,989 WARN [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-02T17:15:41,989 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T17:15:41,996 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.large.0-1733159741990,5,FailOnTimeoutGroup] 2024-12-02T17:15:42,008 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.small.0-1733159741997,5,FailOnTimeoutGroup] 2024-12-02T17:15:42,009 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,013 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T17:15:42,014 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,014 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,069 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(2659): reportForDuty to master=3b3375d1a925,37351,1733159738385 with port=34079, startcode=1733159740078 2024-12-02T17:15:42,070 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(2659): reportForDuty to master=3b3375d1a925,37351,1733159738385 with port=39557, startcode=1733159739502 2024-12-02T17:15:42,089 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3b3375d1a925,34079,1733159740078 2024-12-02T17:15:42,092 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] master.ServerManager(517): Registering regionserver=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:42,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741831_1007 (size=1321) 2024-12-02T17:15:42,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741831_1007 (size=1321) 2024-12-02T17:15:42,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741831_1007 (size=1321) 2024-12-02T17:15:42,097 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T17:15:42,101 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:42,105 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(2659): reportForDuty to master=3b3375d1a925,37351,1733159738385 with port=33117, startcode=1733159739837 2024-12-02T17:15:42,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3b3375d1a925,39557,1733159739502 2024-12-02T17:15:42,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] master.ServerManager(517): Registering regionserver=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:42,117 DEBUG [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:42,117 DEBUG [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43373 2024-12-02T17:15:42,117 DEBUG [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T17:15:42,121 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3b3375d1a925,33117,1733159739837 2024-12-02T17:15:42,121 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] master.ServerManager(517): Registering regionserver=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:42,122 DEBUG [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:42,122 DEBUG [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43373 2024-12-02T17:15:42,122 DEBUG [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T17:15:42,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T17:15:42,131 DEBUG [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:42,131 DEBUG [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43373 2024-12-02T17:15:42,131 DEBUG [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T17:15:42,132 DEBUG [RS:0;3b3375d1a925:39557 {}] zookeeper.ZKUtil(111): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3b3375d1a925,39557,1733159739502 2024-12-02T17:15:42,132 WARN [RS:0;3b3375d1a925:39557 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T17:15:42,132 INFO [RS:0;3b3375d1a925:39557 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T17:15:42,133 DEBUG [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502 2024-12-02T17:15:42,133 DEBUG [RS:2;3b3375d1a925:34079 {}] zookeeper.ZKUtil(111): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3b3375d1a925,34079,1733159740078 2024-12-02T17:15:42,134 WARN [RS:2;3b3375d1a925:34079 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T17:15:42,134 DEBUG [RS:1;3b3375d1a925:33117 {}] zookeeper.ZKUtil(111): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3b3375d1a925,33117,1733159739837 2024-12-02T17:15:42,134 INFO [RS:2;3b3375d1a925:34079 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T17:15:42,134 DEBUG [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,34079,1733159740078 2024-12-02T17:15:42,134 WARN [RS:1;3b3375d1a925:33117 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T17:15:42,135 INFO [RS:1;3b3375d1a925:33117 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T17:15:42,135 DEBUG [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,33117,1733159739837 2024-12-02T17:15:42,146 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3b3375d1a925,39557,1733159739502] 2024-12-02T17:15:42,146 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3b3375d1a925,34079,1733159740078] 2024-12-02T17:15:42,146 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3b3375d1a925,33117,1733159739837] 2024-12-02T17:15:42,177 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T17:15:42,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741832_1008 (size=32) 2024-12-02T17:15:42,180 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T17:15:42,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741832_1008 (size=32) 2024-12-02T17:15:42,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741832_1008 (size=32) 2024-12-02T17:15:42,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:42,191 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T17:15:42,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T17:15:42,209 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T17:15:42,209 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:42,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:42,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T17:15:42,223 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T17:15:42,224 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T17:15:42,230 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T17:15:42,231 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T17:15:42,231 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:42,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:42,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T17:15:42,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T17:15:42,237 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:42,240 INFO [RS:0;3b3375d1a925:39557 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T17:15:42,240 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:42,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T17:15:42,244 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T17:15:42,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:42,245 INFO [RS:2;3b3375d1a925:34079 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T17:15:42,245 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:42,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T17:15:42,248 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740 2024-12-02T17:15:42,250 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740 2024-12-02T17:15:42,253 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T17:15:42,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T17:15:42,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T17:15:42,255 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T17:15:42,260 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T17:15:42,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T17:15:42,261 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,262 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,262 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,262 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,262 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,263 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,263 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0, corePoolSize=2, maxPoolSize=2 2024-12-02T17:15:42,263 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,263 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,263 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,263 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,264 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,264 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,264 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3b3375d1a925:0, corePoolSize=3, maxPoolSize=3 2024-12-02T17:15:42,264 DEBUG [RS:2;3b3375d1a925:34079 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0, corePoolSize=3, maxPoolSize=3 2024-12-02T17:15:42,267 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T17:15:42,269 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T17:15:42,269 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,269 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,269 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,269 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,270 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,270 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,270 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0, corePoolSize=2, maxPoolSize=2 2024-12-02T17:15:42,270 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,270 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,270 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,272 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,272 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,272 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,272 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3b3375d1a925:0, corePoolSize=3, maxPoolSize=3 2024-12-02T17:15:42,273 DEBUG [RS:0;3b3375d1a925:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0, corePoolSize=3, maxPoolSize=3 2024-12-02T17:15:42,273 INFO [RS:1;3b3375d1a925:33117 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T17:15:42,273 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,292 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:42,293 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67630756, jitterRate=0.007776796817779541}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T17:15:42,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733159742186Initializing all the Stores at 1733159742189 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159742189Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159742205 (+16 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159742205Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159742205Cleaning up temporary data from old regions at 1733159742254 (+49 ms)Region opened successfully at 1733159742297 (+43 ms) 2024-12-02T17:15:42,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T17:15:42,297 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T17:15:42,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T17:15:42,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T17:15:42,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T17:15:42,311 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,311 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,311 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,311 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,312 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,312 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,34079,1733159740078-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T17:15:42,321 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T17:15:42,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733159742297Disabling compacts and flushes for region at 1733159742297Disabling writes for close at 1733159742298 (+1 ms)Writing region close event to WAL at 1733159742321 (+23 ms)Closed at 1733159742321 2024-12-02T17:15:42,328 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,329 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,329 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,329 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,329 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,329 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,39557,1733159739502-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T17:15:42,331 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:42,331 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T17:15:42,340 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T17:15:42,342 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T17:15:42,343 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,343 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,343 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,343 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,343 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,343 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,344 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T17:15:42,344 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0, corePoolSize=2, maxPoolSize=2 2024-12-02T17:15:42,344 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,344 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,344 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,344 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T17:15:42,344 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,344 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:15:42,345 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3b3375d1a925:0, corePoolSize=3, maxPoolSize=3 2024-12-02T17:15:42,345 DEBUG [RS:1;3b3375d1a925:33117 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0, corePoolSize=3, maxPoolSize=3 2024-12-02T17:15:42,346 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,34079,1733159740078-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,346 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,347 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.Replication(171): 3b3375d1a925,34079,1733159740078 started 2024-12-02T17:15:42,364 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:42,370 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T17:15:42,390 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,390 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,390 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,390 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,391 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,391 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,33117,1733159739837-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T17:15:42,402 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,402 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T17:15:42,403 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(1482): Serving as 3b3375d1a925,34079,1733159740078, RpcServer on 3b3375d1a925/172.17.0.3:34079, sessionid=0x1009bfce1af0003 2024-12-02T17:15:42,403 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,39557,1733159739502-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,403 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,403 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.Replication(171): 3b3375d1a925,39557,1733159739502 started 2024-12-02T17:15:42,404 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T17:15:42,404 DEBUG [RS:2;3b3375d1a925:34079 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3b3375d1a925,34079,1733159740078 2024-12-02T17:15:42,407 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b3375d1a925,34079,1733159740078' 2024-12-02T17:15:42,407 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T17:15:42,408 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T17:15:42,411 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T17:15:42,411 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T17:15:42,411 DEBUG [RS:2;3b3375d1a925:34079 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3b3375d1a925,34079,1733159740078 2024-12-02T17:15:42,411 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b3375d1a925,34079,1733159740078' 2024-12-02T17:15:42,411 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T17:15:42,412 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T17:15:42,413 DEBUG [RS:2;3b3375d1a925:34079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T17:15:42,413 INFO [RS:2;3b3375d1a925:34079 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T17:15:42,413 INFO [RS:2;3b3375d1a925:34079 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T17:15:42,432 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T17:15:42,432 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,33117,1733159739837-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,432 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,443 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.Replication(171): 3b3375d1a925,33117,1733159739837 started 2024-12-02T17:15:42,480 INFO [RS:0;3b3375d1a925:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,480 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(1482): Serving as 3b3375d1a925,39557,1733159739502, RpcServer on 3b3375d1a925/172.17.0.3:39557, sessionid=0x1009bfce1af0001 2024-12-02T17:15:42,480 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T17:15:42,480 DEBUG [RS:0;3b3375d1a925:39557 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3b3375d1a925,39557,1733159739502 2024-12-02T17:15:42,480 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b3375d1a925,39557,1733159739502' 2024-12-02T17:15:42,481 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T17:15:42,481 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T17:15:42,484 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T17:15:42,484 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T17:15:42,484 DEBUG [RS:0;3b3375d1a925:39557 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3b3375d1a925,39557,1733159739502 2024-12-02T17:15:42,484 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b3375d1a925,39557,1733159739502' 2024-12-02T17:15:42,484 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T17:15:42,488 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T17:15:42,489 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:42,489 DEBUG [RS:0;3b3375d1a925:39557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T17:15:42,489 INFO [RS:0;3b3375d1a925:39557 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T17:15:42,489 INFO [RS:0;3b3375d1a925:39557 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T17:15:42,489 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(1482): Serving as 3b3375d1a925,33117,1733159739837, RpcServer on 3b3375d1a925/172.17.0.3:33117, sessionid=0x1009bfce1af0002 2024-12-02T17:15:42,490 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T17:15:42,494 DEBUG [RS:1;3b3375d1a925:33117 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3b3375d1a925,33117,1733159739837 2024-12-02T17:15:42,494 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b3375d1a925,33117,1733159739837' 2024-12-02T17:15:42,495 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T17:15:42,496 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T17:15:42,497 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T17:15:42,497 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T17:15:42,497 DEBUG [RS:1;3b3375d1a925:33117 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3b3375d1a925,33117,1733159739837 2024-12-02T17:15:42,498 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b3375d1a925,33117,1733159739837' 2024-12-02T17:15:42,498 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T17:15:42,498 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T17:15:42,499 DEBUG [RS:1;3b3375d1a925:33117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T17:15:42,499 INFO [RS:1;3b3375d1a925:33117 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T17:15:42,499 INFO [RS:1;3b3375d1a925:33117 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T17:15:42,520 INFO [RS:2;3b3375d1a925:34079 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T17:15:42,522 WARN [3b3375d1a925:37351 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T17:15:42,524 INFO [RS:2;3b3375d1a925:34079 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b3375d1a925%2C34079%2C1733159740078, suffix=, logDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,34079,1733159740078, archiveDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs, maxLogs=32 2024-12-02T17:15:42,548 DEBUG [RS:2;3b3375d1a925:34079 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,34079,1733159740078/3b3375d1a925%2C34079%2C1733159740078.1733159742528, exclude list is [], retry=0 2024-12-02T17:15:42,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41165,DS-a26ba80d-4012-4587-b072-51db620caa12,DISK] 2024-12-02T17:15:42,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40475,DS-dd8dc82b-5639-4510-9136-c7aeaf6c3351,DISK] 2024-12-02T17:15:42,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38149,DS-668e2095-1fca-40fa-bb76-12a999cf7985,DISK] 2024-12-02T17:15:42,590 INFO [RS:0;3b3375d1a925:39557 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T17:15:42,595 INFO [RS:0;3b3375d1a925:39557 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b3375d1a925%2C39557%2C1733159739502, suffix=, logDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502, archiveDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs, maxLogs=32 2024-12-02T17:15:42,600 INFO [RS:1;3b3375d1a925:33117 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T17:15:42,603 INFO [RS:1;3b3375d1a925:33117 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b3375d1a925%2C33117%2C1733159739837, suffix=, logDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,33117,1733159739837, archiveDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs, maxLogs=32 2024-12-02T17:15:42,628 DEBUG [RS:1;3b3375d1a925:33117 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,33117,1733159739837/3b3375d1a925%2C33117%2C1733159739837.1733159742606, exclude list is [], retry=0 2024-12-02T17:15:42,629 INFO [RS:2;3b3375d1a925:34079 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,34079,1733159740078/3b3375d1a925%2C34079%2C1733159740078.1733159742528 2024-12-02T17:15:42,632 DEBUG [RS:0;3b3375d1a925:39557 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502/3b3375d1a925%2C39557%2C1733159739502.1733159742597, exclude list is [], retry=0 2024-12-02T17:15:42,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41165,DS-a26ba80d-4012-4587-b072-51db620caa12,DISK] 2024-12-02T17:15:42,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38149,DS-668e2095-1fca-40fa-bb76-12a999cf7985,DISK] 2024-12-02T17:15:42,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40475,DS-dd8dc82b-5639-4510-9136-c7aeaf6c3351,DISK] 2024-12-02T17:15:42,639 DEBUG [RS:2;3b3375d1a925:34079 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37663:37663),(127.0.0.1/127.0.0.1:44061:44061),(127.0.0.1/127.0.0.1:46411:46411)] 2024-12-02T17:15:42,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40475,DS-dd8dc82b-5639-4510-9136-c7aeaf6c3351,DISK] 2024-12-02T17:15:42,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38149,DS-668e2095-1fca-40fa-bb76-12a999cf7985,DISK] 2024-12-02T17:15:42,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41165,DS-a26ba80d-4012-4587-b072-51db620caa12,DISK] 2024-12-02T17:15:42,702 INFO [RS:1;3b3375d1a925:33117 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,33117,1733159739837/3b3375d1a925%2C33117%2C1733159739837.1733159742606 2024-12-02T17:15:42,710 INFO [RS:0;3b3375d1a925:39557 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502/3b3375d1a925%2C39557%2C1733159739502.1733159742597 2024-12-02T17:15:42,713 DEBUG [RS:1;3b3375d1a925:33117 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44061:44061),(127.0.0.1/127.0.0.1:37663:37663),(127.0.0.1/127.0.0.1:46411:46411)] 2024-12-02T17:15:42,713 DEBUG [RS:0;3b3375d1a925:39557 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37663:37663),(127.0.0.1/127.0.0.1:46411:46411),(127.0.0.1/127.0.0.1:44061:44061)] 2024-12-02T17:15:42,775 DEBUG [3b3375d1a925:37351 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-02T17:15:42,791 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(204): Hosts are {3b3375d1a925=0} racks are {/default-rack=0} 2024-12-02T17:15:42,802 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T17:15:42,802 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T17:15:42,802 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T17:15:42,802 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T17:15:42,802 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T17:15:42,802 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T17:15:42,802 INFO [3b3375d1a925:37351 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T17:15:42,802 INFO [3b3375d1a925:37351 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T17:15:42,802 INFO [3b3375d1a925:37351 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T17:15:42,802 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T17:15:42,812 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:42,824 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,39557,1733159739502, state=OPENING 2024-12-02T17:15:42,830 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T17:15:42,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:42,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:42,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:42,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:42,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:42,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:42,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:42,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:42,836 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:42,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,39557,1733159739502}] 2024-12-02T17:15:43,024 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T17:15:43,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55577, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T17:15:43,055 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T17:15:43,055 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T17:15:43,056 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T17:15:43,061 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b3375d1a925%2C39557%2C1733159739502.meta, suffix=.meta, logDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502, archiveDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs, maxLogs=32 2024-12-02T17:15:43,085 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, exclude list is [], retry=0 2024-12-02T17:15:43,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41165,DS-a26ba80d-4012-4587-b072-51db620caa12,DISK] 2024-12-02T17:15:43,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38149,DS-668e2095-1fca-40fa-bb76-12a999cf7985,DISK] 2024-12-02T17:15:43,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40475,DS-dd8dc82b-5639-4510-9136-c7aeaf6c3351,DISK] 2024-12-02T17:15:43,104 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta 2024-12-02T17:15:43,110 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44061:44061),(127.0.0.1/127.0.0.1:46411:46411),(127.0.0.1/127.0.0.1:37663:37663)] 2024-12-02T17:15:43,110 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:15:43,113 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T17:15:43,116 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T17:15:43,126 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T17:15:43,138 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T17:15:43,139 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:43,139 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T17:15:43,139 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T17:15:43,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T17:15:43,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T17:15:43,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:43,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:43,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T17:15:43,168 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T17:15:43,168 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:43,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:43,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T17:15:43,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T17:15:43,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:43,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:43,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T17:15:43,175 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T17:15:43,175 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:43,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:43,176 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T17:15:43,178 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740 2024-12-02T17:15:43,181 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740 2024-12-02T17:15:43,188 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T17:15:43,188 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T17:15:43,189 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T17:15:43,193 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T17:15:43,196 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64035238, jitterRate=-0.045800596475601196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T17:15:43,196 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T17:15:43,198 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733159743140Writing region info on filesystem at 1733159743140Initializing all the Stores at 1733159743143 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159743143Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159743161 (+18 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159743161Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159743161Cleaning up temporary data from old regions at 1733159743188 (+27 ms)Running coprocessor post-open hooks at 1733159743197 (+9 ms)Region opened successfully at 1733159743198 (+1 ms) 2024-12-02T17:15:43,210 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733159743012 2024-12-02T17:15:43,228 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T17:15:43,229 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T17:15:43,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:43,232 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,39557,1733159739502, state=OPEN 2024-12-02T17:15:43,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:43,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:43,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:43,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:43,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:43,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:43,237 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:43,237 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:43,238 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:43,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T17:15:43,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,39557,1733159739502 in 400 msec 2024-12-02T17:15:43,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T17:15:43,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 907 msec 2024-12-02T17:15:43,268 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:43,272 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T17:15:43,307 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:43,311 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,39557,1733159739502, seqNum=-1] 2024-12-02T17:15:43,349 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:43,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35449, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:43,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6850 sec 2024-12-02T17:15:43,396 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733159743395, completionTime=-1 2024-12-02T17:15:43,400 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-02T17:15:43,400 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:43,437 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-02T17:15:43,437 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159803437 2024-12-02T17:15:43,437 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159863437 2024-12-02T17:15:43,437 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-12-02T17:15:43,439 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:43,449 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,37351,1733159738385-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:43,451 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,37351,1733159738385-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:43,451 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,37351,1733159738385-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:43,453 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3b3375d1a925:37351, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:43,453 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:43,462 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:43,465 DEBUG [master/3b3375d1a925:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T17:15:43,579 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.309sec 2024-12-02T17:15:43,599 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T17:15:43,601 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T17:15:43,602 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T17:15:43,603 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T17:15:43,603 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T17:15:43,604 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,37351,1733159738385-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T17:15:43,606 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,37351,1733159738385-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T17:15:43,614 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T17:15:43,615 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T17:15:43,616 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,37351,1733159738385-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:15:43,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359847dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:43,684 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T17:15:43,684 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T17:15:43,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,37351,-1 for getting cluster id 2024-12-02T17:15:43,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:43,706 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8980728-5223-41da-bed8-9a75b45a5f07' 2024-12-02T17:15:43,709 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:43,710 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8980728-5223-41da-bed8-9a75b45a5f07" 2024-12-02T17:15:43,712 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68205ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:43,713 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,37351,-1] 2024-12-02T17:15:43,716 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:43,718 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:43,719 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:43,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b221f9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:43,732 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:43,741 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,39557,1733159739502, seqNum=-1] 2024-12-02T17:15:43,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:43,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:43,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3b3375d1a925,37351,1733159738385 2024-12-02T17:15:43,799 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T17:15:43,806 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3b3375d1a925,37351,1733159738385 2024-12-02T17:15:43,809 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b7a199c 2024-12-02T17:15:43,811 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T17:15:43,836 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T17:15:43,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:15:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:43,863 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T17:15:43,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-02T17:15:43,866 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:43,869 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T17:15:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:15:43,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741837_1013 (size=358) 2024-12-02T17:15:43,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741837_1013 (size=358) 2024-12-02T17:15:43,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741837_1013 (size=358) 2024-12-02T17:15:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:15:44,006 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 16d4fe4d0593f960b6c0f8237bf0711b, NAME => 'test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,015 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fc881fc21c14f92b22d6d3f233ac48b1, NAME => 'test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,018 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => bf8dc4ff69f7afd79a39737b7174af90, NAME => 'test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741838_1014 (size=45) 2024-12-02T17:15:44,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741838_1014 (size=45) 2024-12-02T17:15:44,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741838_1014 (size=45) 2024-12-02T17:15:44,129 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,129 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fc881fc21c14f92b22d6d3f233ac48b1, disabling compactions & flushes 2024-12-02T17:15:44,129 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:44,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:44,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. after waiting 0 ms 2024-12-02T17:15:44,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:44,130 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:44,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fc881fc21c14f92b22d6d3f233ac48b1: Waiting for close lock at 1733159744129Disabling compacts and flushes for region at 1733159744129Disabling writes for close at 1733159744130 (+1 ms)Writing region close event to WAL at 1733159744130Closed at 1733159744130 2024-12-02T17:15:44,132 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ee07072225557a5ffccbd5b4dc3f8250, NAME => 'test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741839_1015 (size=42) 2024-12-02T17:15:44,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741839_1015 (size=42) 2024-12-02T17:15:44,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741839_1015 (size=42) 2024-12-02T17:15:44,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741840_1016 (size=45) 2024-12-02T17:15:44,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741840_1016 (size=45) 2024-12-02T17:15:44,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741840_1016 (size=45) 2024-12-02T17:15:44,145 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,145 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing bf8dc4ff69f7afd79a39737b7174af90, disabling compactions & flushes 2024-12-02T17:15:44,145 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:44,145 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:44,145 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. after waiting 0 ms 2024-12-02T17:15:44,145 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:44,145 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:44,146 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for bf8dc4ff69f7afd79a39737b7174af90: Waiting for close lock at 1733159744145Disabling compacts and flushes for region at 1733159744145Disabling writes for close at 1733159744145Writing region close event to WAL at 1733159744145Closed at 1733159744145 2024-12-02T17:15:44,146 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8981ccd8f09f082161872d7b109756f0, NAME => 'test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741842_1018 (size=45) 2024-12-02T17:15:44,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741842_1018 (size=45) 2024-12-02T17:15:44,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741842_1018 (size=45) 2024-12-02T17:15:44,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8981ccd8f09f082161872d7b109756f0, disabling compactions & flushes 2024-12-02T17:15:44,194 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:44,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:44,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. after waiting 0 ms 2024-12-02T17:15:44,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:44,194 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:44,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8981ccd8f09f082161872d7b109756f0: Waiting for close lock at 1733159744194Disabling compacts and flushes for region at 1733159744194Disabling writes for close at 1733159744194Writing region close event to WAL at 1733159744194Closed at 1733159744194 2024-12-02T17:15:44,195 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 846f82385af02c9a3c05149f7ca40c3a, NAME => 'test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:15:44,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741841_1017 (size=45) 2024-12-02T17:15:44,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741841_1017 (size=45) 2024-12-02T17:15:44,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741841_1017 (size=45) 2024-12-02T17:15:44,227 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ee07072225557a5ffccbd5b4dc3f8250, disabling compactions & flushes 2024-12-02T17:15:44,228 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:44,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:44,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. after waiting 0 ms 2024-12-02T17:15:44,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:44,228 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:44,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ee07072225557a5ffccbd5b4dc3f8250: Waiting for close lock at 1733159744228Disabling compacts and flushes for region at 1733159744228Disabling writes for close at 1733159744228Writing region close event to WAL at 1733159744228Closed at 1733159744228 2024-12-02T17:15:44,229 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 76faba7cbca7c506fec4bb6f534477a2, NAME => 'test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741843_1019 (size=45) 2024-12-02T17:15:44,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741843_1019 (size=45) 2024-12-02T17:15:44,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741843_1019 (size=45) 2024-12-02T17:15:44,255 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,255 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 846f82385af02c9a3c05149f7ca40c3a, disabling compactions & flushes 2024-12-02T17:15:44,255 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:44,255 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:44,255 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. after waiting 0 ms 2024-12-02T17:15:44,255 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:44,255 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:44,255 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 846f82385af02c9a3c05149f7ca40c3a: Waiting for close lock at 1733159744255Disabling compacts and flushes for region at 1733159744255Disabling writes for close at 1733159744255Writing region close event to WAL at 1733159744255Closed at 1733159744255 2024-12-02T17:15:44,257 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cf990684b331d6bf1770501628bfa34f, NAME => 'test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741844_1020 (size=45) 2024-12-02T17:15:44,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741844_1020 (size=45) 2024-12-02T17:15:44,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741844_1020 (size=45) 2024-12-02T17:15:44,272 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,273 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 76faba7cbca7c506fec4bb6f534477a2, disabling compactions & flushes 2024-12-02T17:15:44,273 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:44,273 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:44,273 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. after waiting 0 ms 2024-12-02T17:15:44,273 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:44,273 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:44,273 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 76faba7cbca7c506fec4bb6f534477a2: Waiting for close lock at 1733159744272Disabling compacts and flushes for region at 1733159744273 (+1 ms)Disabling writes for close at 1733159744273Writing region close event to WAL at 1733159744273Closed at 1733159744273 2024-12-02T17:15:44,274 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => eba2814ba0cf502df62df55d77023b23, NAME => 'test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741845_1021 (size=45) 2024-12-02T17:15:44,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741845_1021 (size=45) 2024-12-02T17:15:44,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741845_1021 (size=45) 2024-12-02T17:15:44,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cf990684b331d6bf1770501628bfa34f, disabling compactions & flushes 2024-12-02T17:15:44,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:44,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:44,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. after waiting 0 ms 2024-12-02T17:15:44,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:44,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:44,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cf990684b331d6bf1770501628bfa34f: Waiting for close lock at 1733159744313Disabling compacts and flushes for region at 1733159744313Disabling writes for close at 1733159744313Writing region close event to WAL at 1733159744313Closed at 1733159744313 2024-12-02T17:15:44,314 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 77b4e8908ad2baccd7eb5d5fa846dff8, NAME => 'test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741846_1022 (size=45) 2024-12-02T17:15:44,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741846_1022 (size=45) 2024-12-02T17:15:44,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741846_1022 (size=45) 2024-12-02T17:15:44,329 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,329 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing eba2814ba0cf502df62df55d77023b23, disabling compactions & flushes 2024-12-02T17:15:44,329 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:44,329 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:44,329 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. after waiting 0 ms 2024-12-02T17:15:44,329 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:44,329 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:44,329 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for eba2814ba0cf502df62df55d77023b23: Waiting for close lock at 1733159744329Disabling compacts and flushes for region at 1733159744329Disabling writes for close at 1733159744329Writing region close event to WAL at 1733159744329Closed at 1733159744329 2024-12-02T17:15:44,330 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 31473ece20fde70fc9741ad3ff42aadc, NAME => 'test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741848_1024 (size=45) 2024-12-02T17:15:44,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741848_1024 (size=45) 2024-12-02T17:15:44,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741848_1024 (size=45) 2024-12-02T17:15:44,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 31473ece20fde70fc9741ad3ff42aadc, disabling compactions & flushes 2024-12-02T17:15:44,354 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:44,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:44,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. after waiting 0 ms 2024-12-02T17:15:44,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:44,354 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:44,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 31473ece20fde70fc9741ad3ff42aadc: Waiting for close lock at 1733159744354Disabling compacts and flushes for region at 1733159744354Disabling writes for close at 1733159744354Writing region close event to WAL at 1733159744354Closed at 1733159744354 2024-12-02T17:15:44,355 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 80be845edde76a33d61215cef1be07d7, NAME => 'test,lll,1733159743838.80be845edde76a33d61215cef1be07d7.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741847_1023 (size=45) 2024-12-02T17:15:44,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741847_1023 (size=45) 2024-12-02T17:15:44,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741847_1023 (size=45) 2024-12-02T17:15:44,382 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,383 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 77b4e8908ad2baccd7eb5d5fa846dff8, disabling compactions & flushes 2024-12-02T17:15:44,383 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:44,383 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:44,383 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. after waiting 0 ms 2024-12-02T17:15:44,383 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:44,383 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:44,383 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 77b4e8908ad2baccd7eb5d5fa846dff8: Waiting for close lock at 1733159744383Disabling compacts and flushes for region at 1733159744383Disabling writes for close at 1733159744383Writing region close event to WAL at 1733159744383Closed at 1733159744383 2024-12-02T17:15:44,384 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ba8c728daa66bac733cfa12bcd38a313, NAME => 'test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741849_1025 (size=45) 2024-12-02T17:15:44,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741849_1025 (size=45) 2024-12-02T17:15:44,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741849_1025 (size=45) 2024-12-02T17:15:44,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1733159743838.80be845edde76a33d61215cef1be07d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 80be845edde76a33d61215cef1be07d7, disabling compactions & flushes 2024-12-02T17:15:44,409 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:44,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:44,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. after waiting 0 ms 2024-12-02T17:15:44,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:44,409 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:44,410 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 80be845edde76a33d61215cef1be07d7: Waiting for close lock at 1733159744409Disabling compacts and flushes for region at 1733159744409Disabling writes for close at 1733159744409Writing region close event to WAL at 1733159744409Closed at 1733159744409 2024-12-02T17:15:44,410 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e8a17d43fe45538f8e98a951b88f67de, NAME => 'test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741850_1026 (size=45) 2024-12-02T17:15:44,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741850_1026 (size=45) 2024-12-02T17:15:44,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741850_1026 (size=45) 2024-12-02T17:15:44,424 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,425 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ba8c728daa66bac733cfa12bcd38a313, disabling compactions & flushes 2024-12-02T17:15:44,425 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:44,425 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:44,425 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. after waiting 0 ms 2024-12-02T17:15:44,425 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:44,425 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:44,425 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ba8c728daa66bac733cfa12bcd38a313: Waiting for close lock at 1733159744424Disabling compacts and flushes for region at 1733159744424Disabling writes for close at 1733159744425 (+1 ms)Writing region close event to WAL at 1733159744425Closed at 1733159744425 2024-12-02T17:15:44,425 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6dd20102809dc64a92201c4ba1e20e38, NAME => 'test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741851_1027 (size=45) 2024-12-02T17:15:44,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741851_1027 (size=45) 2024-12-02T17:15:44,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741851_1027 (size=45) 2024-12-02T17:15:44,450 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,450 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e8a17d43fe45538f8e98a951b88f67de, disabling compactions & flushes 2024-12-02T17:15:44,450 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:44,450 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:44,450 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. after waiting 0 ms 2024-12-02T17:15:44,450 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:44,450 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:44,450 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e8a17d43fe45538f8e98a951b88f67de: Waiting for close lock at 1733159744450Disabling compacts and flushes for region at 1733159744450Disabling writes for close at 1733159744450Writing region close event to WAL at 1733159744450Closed at 1733159744450 2024-12-02T17:15:44,451 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d612ab016a94fed4cbfb22a0c74d8f6a, NAME => 'test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741852_1028 (size=45) 2024-12-02T17:15:44,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741852_1028 (size=45) 2024-12-02T17:15:44,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741852_1028 (size=45) 2024-12-02T17:15:44,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6dd20102809dc64a92201c4ba1e20e38, disabling compactions & flushes 2024-12-02T17:15:44,458 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:44,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:44,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. after waiting 0 ms 2024-12-02T17:15:44,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:44,458 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:44,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6dd20102809dc64a92201c4ba1e20e38: Waiting for close lock at 1733159744458Disabling compacts and flushes for region at 1733159744458Disabling writes for close at 1733159744458Writing region close event to WAL at 1733159744458Closed at 1733159744458 2024-12-02T17:15:44,459 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0fd047fbb6ae4c8ccd6da6584c52f6db, NAME => 'test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741853_1029 (size=45) 2024-12-02T17:15:44,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741853_1029 (size=45) 2024-12-02T17:15:44,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741853_1029 (size=45) 2024-12-02T17:15:44,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d612ab016a94fed4cbfb22a0c74d8f6a, disabling compactions & flushes 2024-12-02T17:15:44,519 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:44,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:44,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. after waiting 0 ms 2024-12-02T17:15:44,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:44,519 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:44,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d612ab016a94fed4cbfb22a0c74d8f6a: Waiting for close lock at 1733159744519Disabling compacts and flushes for region at 1733159744519Disabling writes for close at 1733159744519Writing region close event to WAL at 1733159744519Closed at 1733159744519 2024-12-02T17:15:44,520 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => af5ee9a539ef801e1148eebca0987b9c, NAME => 'test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:15:44,534 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,534 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 16d4fe4d0593f960b6c0f8237bf0711b, disabling compactions & flushes 2024-12-02T17:15:44,534 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:44,534 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:44,534 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. after waiting 0 ms 2024-12-02T17:15:44,534 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:44,534 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:44,535 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 16d4fe4d0593f960b6c0f8237bf0711b: Waiting for close lock at 1733159744534Disabling compacts and flushes for region at 1733159744534Disabling writes for close at 1733159744534Writing region close event to WAL at 1733159744534Closed at 1733159744534 2024-12-02T17:15:44,535 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a37d5d00fe28247586762a18d6e0943f, NAME => 'test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741854_1030 (size=45) 2024-12-02T17:15:44,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741854_1030 (size=45) 2024-12-02T17:15:44,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741854_1030 (size=45) 2024-12-02T17:15:44,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0fd047fbb6ae4c8ccd6da6584c52f6db, disabling compactions & flushes 2024-12-02T17:15:44,548 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:44,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:44,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. after waiting 0 ms 2024-12-02T17:15:44,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:44,548 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:44,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0fd047fbb6ae4c8ccd6da6584c52f6db: Waiting for close lock at 1733159744548Disabling compacts and flushes for region at 1733159744548Disabling writes for close at 1733159744548Writing region close event to WAL at 1733159744548Closed at 1733159744548 2024-12-02T17:15:44,549 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8cd24aed41e647f8e01f1c4bdbf2c148, NAME => 'test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741856_1032 (size=45) 2024-12-02T17:15:44,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741856_1032 (size=45) 2024-12-02T17:15:44,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741856_1032 (size=45) 2024-12-02T17:15:44,598 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,598 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a37d5d00fe28247586762a18d6e0943f, disabling compactions & flushes 2024-12-02T17:15:44,598 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:44,598 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:44,598 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. after waiting 0 ms 2024-12-02T17:15:44,598 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:44,598 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:44,599 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a37d5d00fe28247586762a18d6e0943f: Waiting for close lock at 1733159744598Disabling compacts and flushes for region at 1733159744598Disabling writes for close at 1733159744598Writing region close event to WAL at 1733159744598Closed at 1733159744598 2024-12-02T17:15:44,600 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4eb95452ea2c6ad04c7114e1401424b9, NAME => 'test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741855_1031 (size=45) 2024-12-02T17:15:44,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741855_1031 (size=45) 2024-12-02T17:15:44,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741857_1033 (size=45) 2024-12-02T17:15:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741857_1033 (size=45) 2024-12-02T17:15:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741855_1031 (size=45) 2024-12-02T17:15:44,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741857_1033 (size=45) 2024-12-02T17:15:44,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741858_1034 (size=45) 2024-12-02T17:15:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741858_1034 (size=45) 2024-12-02T17:15:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741858_1034 (size=45) 2024-12-02T17:15:44,637 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,637 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4eb95452ea2c6ad04c7114e1401424b9, disabling compactions & flushes 2024-12-02T17:15:44,637 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:44,637 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:44,638 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. after waiting 0 ms 2024-12-02T17:15:44,638 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:44,638 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:44,638 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4eb95452ea2c6ad04c7114e1401424b9: Waiting for close lock at 1733159744637Disabling compacts and flushes for region at 1733159744637Disabling writes for close at 1733159744638 (+1 ms)Writing region close event to WAL at 1733159744638Closed at 1733159744638 2024-12-02T17:15:44,639 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1fd27a546540e9b8b8f7a078f3515500, NAME => 'test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741859_1035 (size=45) 2024-12-02T17:15:44,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741859_1035 (size=45) 2024-12-02T17:15:44,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741859_1035 (size=45) 2024-12-02T17:15:44,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:44,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1fd27a546540e9b8b8f7a078f3515500, disabling compactions & flushes 2024-12-02T17:15:44,658 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:44,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:44,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. after waiting 0 ms 2024-12-02T17:15:44,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:44,658 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:44,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1fd27a546540e9b8b8f7a078f3515500: Waiting for close lock at 1733159744658Disabling compacts and flushes for region at 1733159744658Disabling writes for close at 1733159744658Writing region close event to WAL at 1733159744658Closed at 1733159744658 2024-12-02T17:15:44,659 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 51e940693c5f493c59a469d01db516d5, NAME => 'test,www,1733159743838.51e940693c5f493c59a469d01db516d5.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:44,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741860_1036 (size=45) 2024-12-02T17:15:44,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741860_1036 (size=45) 2024-12-02T17:15:44,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741860_1036 (size=45) 2024-12-02T17:15:45,006 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:45,006 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing af5ee9a539ef801e1148eebca0987b9c, disabling compactions & flushes 2024-12-02T17:15:45,006 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:45,006 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:45,006 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. after waiting 0 ms 2024-12-02T17:15:45,007 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:45,007 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:45,007 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for af5ee9a539ef801e1148eebca0987b9c: Waiting for close lock at 1733159745006Disabling compacts and flushes for region at 1733159745006Disabling writes for close at 1733159745007 (+1 ms)Writing region close event to WAL at 1733159745007Closed at 1733159745007 2024-12-02T17:15:45,007 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d53e8d53f3fcf0c3420445811190e425, NAME => 'test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:45,011 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:45,012 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8cd24aed41e647f8e01f1c4bdbf2c148, disabling compactions & flushes 2024-12-02T17:15:45,012 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:45,012 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:45,012 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. after waiting 0 ms 2024-12-02T17:15:45,012 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:45,012 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:45,012 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8cd24aed41e647f8e01f1c4bdbf2c148: Waiting for close lock at 1733159745012Disabling compacts and flushes for region at 1733159745012Disabling writes for close at 1733159745012Writing region close event to WAL at 1733159745012Closed at 1733159745012 2024-12-02T17:15:45,013 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3882d8b2ab81cfedea2b9eda817f13bf, NAME => 'test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:15:45,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741861_1037 (size=45) 2024-12-02T17:15:45,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741861_1037 (size=45) 2024-12-02T17:15:45,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741861_1037 (size=45) 2024-12-02T17:15:45,084 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:45,084 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 3882d8b2ab81cfedea2b9eda817f13bf, disabling compactions & flushes 2024-12-02T17:15:45,084 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:45,084 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:45,084 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. after waiting 0 ms 2024-12-02T17:15:45,084 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:45,084 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:45,084 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3882d8b2ab81cfedea2b9eda817f13bf: Waiting for close lock at 1733159745084Disabling compacts and flushes for region at 1733159745084Disabling writes for close at 1733159745084Writing region close event to WAL at 1733159745084Closed at 1733159745084 2024-12-02T17:15:45,085 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5b72b8ea0bceb0151b39bea3aa7f2a9e, NAME => 'test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567 2024-12-02T17:15:45,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741862_1038 (size=45) 2024-12-02T17:15:45,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741862_1038 (size=45) 2024-12-02T17:15:45,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741862_1038 (size=45) 2024-12-02T17:15:45,091 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:45,091 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d53e8d53f3fcf0c3420445811190e425, disabling compactions & flushes 2024-12-02T17:15:45,092 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:45,092 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:45,092 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. after waiting 0 ms 2024-12-02T17:15:45,092 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:45,092 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:45,092 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d53e8d53f3fcf0c3420445811190e425: Waiting for close lock at 1733159745091Disabling compacts and flushes for region at 1733159745091Disabling writes for close at 1733159745092 (+1 ms)Writing region close event to WAL at 1733159745092Closed at 1733159745092 2024-12-02T17:15:45,103 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1733159743838.51e940693c5f493c59a469d01db516d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:45,103 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 51e940693c5f493c59a469d01db516d5, disabling compactions & flushes 2024-12-02T17:15:45,103 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:45,103 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:45,103 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733159743838.51e940693c5f493c59a469d01db516d5. after waiting 0 ms 2024-12-02T17:15:45,103 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:45,103 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:45,103 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 51e940693c5f493c59a469d01db516d5: Waiting for close lock at 1733159745103Disabling compacts and flushes for region at 1733159745103Disabling writes for close at 1733159745103Writing region close event to WAL at 1733159745103Closed at 1733159745103 2024-12-02T17:15:45,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741863_1039 (size=42) 2024-12-02T17:15:45,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741863_1039 (size=42) 2024-12-02T17:15:45,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741863_1039 (size=42) 2024-12-02T17:15:45,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:45,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5b72b8ea0bceb0151b39bea3aa7f2a9e, disabling compactions & flushes 2024-12-02T17:15:45,158 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:45,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:45,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. after waiting 0 ms 2024-12-02T17:15:45,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:45,158 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:45,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5b72b8ea0bceb0151b39bea3aa7f2a9e: Waiting for close lock at 1733159745158Disabling compacts and flushes for region at 1733159745158Disabling writes for close at 1733159745158Writing region close event to WAL at 1733159745158Closed at 1733159745158 2024-12-02T17:15:45,164 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T17:15:45,172 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733159743838.80be845edde76a33d61215cef1be07d7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733159743838.51e940693c5f493c59a469d01db516d5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733159745164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159745164"}]},"ts":"1733159745164"} 2024-12-02T17:15:45,296 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-02T17:15:45,309 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T17:15:45,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159745309"}]},"ts":"1733159745309"} 2024-12-02T17:15:45,337 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-02T17:15:45,348 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {3b3375d1a925=0} racks are {/default-rack=0} 2024-12-02T17:15:45,409 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T17:15:45,409 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T17:15:45,409 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T17:15:45,409 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T17:15:45,409 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T17:15:45,409 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T17:15:45,409 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T17:15:45,409 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T17:15:45,409 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T17:15:45,409 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T17:15:45,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=16d4fe4d0593f960b6c0f8237bf0711b, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc881fc21c14f92b22d6d3f233ac48b1, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf8dc4ff69f7afd79a39737b7174af90, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee07072225557a5ffccbd5b4dc3f8250, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=846f82385af02c9a3c05149f7ca40c3a, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cf990684b331d6bf1770501628bfa34f, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eba2814ba0cf502df62df55d77023b23, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77b4e8908ad2baccd7eb5d5fa846dff8, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80be845edde76a33d61215cef1be07d7, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e8a17d43fe45538f8e98a951b88f67de, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6dd20102809dc64a92201c4ba1e20e38, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0fd047fbb6ae4c8ccd6da6584c52f6db, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8cd24aed41e647f8e01f1c4bdbf2c148, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4eb95452ea2c6ad04c7114e1401424b9, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1fd27a546540e9b8b8f7a078f3515500, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51e940693c5f493c59a469d01db516d5, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3882d8b2ab81cfedea2b9eda817f13bf, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5b72b8ea0bceb0151b39bea3aa7f2a9e, ASSIGN}] 2024-12-02T17:15:45,438 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5b72b8ea0bceb0151b39bea3aa7f2a9e, ASSIGN 2024-12-02T17:15:45,440 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5b72b8ea0bceb0151b39bea3aa7f2a9e, ASSIGN; state=OFFLINE, location=3b3375d1a925,33117,1733159739837; forceNewPlan=false, retain=false 2024-12-02T17:15:45,442 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3882d8b2ab81cfedea2b9eda817f13bf, ASSIGN 2024-12-02T17:15:45,443 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3882d8b2ab81cfedea2b9eda817f13bf, ASSIGN; state=OFFLINE, location=3b3375d1a925,33117,1733159739837; forceNewPlan=false, retain=false 2024-12-02T17:15:45,445 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:45,447 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN; state=OFFLINE, location=3b3375d1a925,39557,1733159739502; forceNewPlan=false, retain=false 2024-12-02T17:15:45,449 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51e940693c5f493c59a469d01db516d5, ASSIGN 2024-12-02T17:15:45,452 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=51e940693c5f493c59a469d01db516d5, ASSIGN; state=OFFLINE, location=3b3375d1a925,34079,1733159740078; forceNewPlan=false, retain=false 2024-12-02T17:15:45,454 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1fd27a546540e9b8b8f7a078f3515500, ASSIGN 2024-12-02T17:15:45,455 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1fd27a546540e9b8b8f7a078f3515500, ASSIGN; state=OFFLINE, location=3b3375d1a925,33117,1733159739837; forceNewPlan=false, retain=false 2024-12-02T17:15:45,460 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4eb95452ea2c6ad04c7114e1401424b9, ASSIGN 2024-12-02T17:15:45,462 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4eb95452ea2c6ad04c7114e1401424b9, ASSIGN; state=OFFLINE, location=3b3375d1a925,34079,1733159740078; forceNewPlan=false, retain=false 2024-12-02T17:15:45,464 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8cd24aed41e647f8e01f1c4bdbf2c148, ASSIGN 2024-12-02T17:15:45,466 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8cd24aed41e647f8e01f1c4bdbf2c148, ASSIGN; state=OFFLINE, location=3b3375d1a925,34079,1733159740078; forceNewPlan=false, retain=false 2024-12-02T17:15:45,467 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:45,469 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN; state=OFFLINE, location=3b3375d1a925,39557,1733159739502; forceNewPlan=false, retain=false 2024-12-02T17:15:45,471 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:45,481 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN; state=OFFLINE, location=3b3375d1a925,39557,1733159739502; forceNewPlan=false, retain=false 2024-12-02T17:15:45,485 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0fd047fbb6ae4c8ccd6da6584c52f6db, ASSIGN 2024-12-02T17:15:45,488 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0fd047fbb6ae4c8ccd6da6584c52f6db, ASSIGN; state=OFFLINE, location=3b3375d1a925,33117,1733159739837; forceNewPlan=false, retain=false 2024-12-02T17:15:45,513 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:45,516 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN; state=OFFLINE, location=3b3375d1a925,39557,1733159739502; forceNewPlan=false, retain=false 2024-12-02T17:15:45,519 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6dd20102809dc64a92201c4ba1e20e38, ASSIGN 2024-12-02T17:15:45,521 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6dd20102809dc64a92201c4ba1e20e38, ASSIGN; state=OFFLINE, location=3b3375d1a925,33117,1733159739837; forceNewPlan=false, retain=false 2024-12-02T17:15:45,526 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e8a17d43fe45538f8e98a951b88f67de, ASSIGN 2024-12-02T17:15:45,534 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e8a17d43fe45538f8e98a951b88f67de, ASSIGN; state=OFFLINE, location=3b3375d1a925,34079,1733159740078; forceNewPlan=false, retain=false 2024-12-02T17:15:45,536 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:45,542 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN; state=OFFLINE, location=3b3375d1a925,39557,1733159739502; forceNewPlan=false, retain=false 2024-12-02T17:15:45,543 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80be845edde76a33d61215cef1be07d7, ASSIGN 2024-12-02T17:15:45,545 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=80be845edde76a33d61215cef1be07d7, ASSIGN; state=OFFLINE, location=3b3375d1a925,33117,1733159739837; forceNewPlan=false, retain=false 2024-12-02T17:15:45,547 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:45,549 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN; state=OFFLINE, location=3b3375d1a925,39557,1733159739502; forceNewPlan=false, retain=false 2024-12-02T17:15:45,551 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77b4e8908ad2baccd7eb5d5fa846dff8, ASSIGN 2024-12-02T17:15:45,554 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=77b4e8908ad2baccd7eb5d5fa846dff8, ASSIGN; state=OFFLINE, location=3b3375d1a925,34079,1733159740078; forceNewPlan=false, retain=false 2024-12-02T17:15:45,557 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eba2814ba0cf502df62df55d77023b23, ASSIGN 2024-12-02T17:15:45,559 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eba2814ba0cf502df62df55d77023b23, ASSIGN; state=OFFLINE, location=3b3375d1a925,33117,1733159739837; forceNewPlan=false, retain=false 2024-12-02T17:15:45,561 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cf990684b331d6bf1770501628bfa34f, ASSIGN 2024-12-02T17:15:45,565 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cf990684b331d6bf1770501628bfa34f, ASSIGN; state=OFFLINE, location=3b3375d1a925,34079,1733159740078; forceNewPlan=false, retain=false 2024-12-02T17:15:45,570 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:45,574 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN; state=OFFLINE, location=3b3375d1a925,39557,1733159739502; forceNewPlan=false, retain=false 2024-12-02T17:15:45,579 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=846f82385af02c9a3c05149f7ca40c3a, ASSIGN 2024-12-02T17:15:45,585 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=846f82385af02c9a3c05149f7ca40c3a, ASSIGN; state=OFFLINE, location=3b3375d1a925,33117,1733159739837; forceNewPlan=false, retain=false 2024-12-02T17:15:45,593 INFO [3b3375d1a925:37351 {}] balancer.BaseLoadBalancer(388): Reassigned 21 regions. 21 retained the pre-restart assignment. 2024-12-02T17:15:45,594 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:45,596 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:45,598 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:45,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:45,600 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:45,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:45,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN because future has completed 2024-12-02T17:15:45,606 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:45,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN because future has completed 2024-12-02T17:15:45,606 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=51e940693c5f493c59a469d01db516d5, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:45,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure d53e8d53f3fcf0c3420445811190e425, server=3b3375d1a925,39557,1733159739502}] 2024-12-02T17:15:45,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN because future has completed 2024-12-02T17:15:45,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN because future has completed 2024-12-02T17:15:45,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN because future has completed 2024-12-02T17:15:45,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN because future has completed 2024-12-02T17:15:45,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN because future has completed 2024-12-02T17:15:45,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=51e940693c5f493c59a469d01db516d5, ASSIGN because future has completed 2024-12-02T17:15:45,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,39557,1733159739502}] 2024-12-02T17:15:45,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51e940693c5f493c59a469d01db516d5, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:45,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,39557,1733159739502}] 2024-12-02T17:15:45,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,39557,1733159739502}] 2024-12-02T17:15:45,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,39557,1733159739502}] 2024-12-02T17:15:45,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,39557,1733159739502}] 2024-12-02T17:15:45,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,39557,1733159739502}] 2024-12-02T17:15:45,800 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:45,800 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:45,802 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=e8a17d43fe45538f8e98a951b88f67de, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:45,804 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:45,805 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=cf990684b331d6bf1770501628bfa34f, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:45,806 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:45,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4eb95452ea2c6ad04c7114e1401424b9, ASSIGN because future has completed 2024-12-02T17:15:45,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4eb95452ea2c6ad04c7114e1401424b9, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:45,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8cd24aed41e647f8e01f1c4bdbf2c148, ASSIGN because future has completed 2024-12-02T17:15:45,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e8a17d43fe45538f8e98a951b88f67de, ASSIGN because future has completed 2024-12-02T17:15:45,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=77b4e8908ad2baccd7eb5d5fa846dff8, ASSIGN because future has completed 2024-12-02T17:15:45,842 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:45,842 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => af5ee9a539ef801e1148eebca0987b9c, NAME => 'test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-02T17:15:45,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77b4e8908ad2baccd7eb5d5fa846dff8, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:45,843 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,844 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:45,844 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,844 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cf990684b331d6bf1770501628bfa34f, ASSIGN because future has completed 2024-12-02T17:15:45,868 INFO [StoreOpener-af5ee9a539ef801e1148eebca0987b9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,869 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T17:15:45,873 INFO [StoreOpener-af5ee9a539ef801e1148eebca0987b9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af5ee9a539ef801e1148eebca0987b9c columnFamilyName family 2024-12-02T17:15:45,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55603, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T17:15:45,873 DEBUG [StoreOpener-af5ee9a539ef801e1148eebca0987b9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:45,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5b72b8ea0bceb0151b39bea3aa7f2a9e, ASSIGN because future has completed 2024-12-02T17:15:45,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b72b8ea0bceb0151b39bea3aa7f2a9e, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:45,880 INFO [StoreOpener-af5ee9a539ef801e1148eebca0987b9c-1 {}] regionserver.HStore(327): Store=af5ee9a539ef801e1148eebca0987b9c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:45,880 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,882 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,882 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,883 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,883 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf990684b331d6bf1770501628bfa34f, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:45,887 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8a17d43fe45538f8e98a951b88f67de, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:45,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cd24aed41e647f8e01f1c4bdbf2c148, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:45,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:45,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=1fd27a546540e9b8b8f7a078f3515500, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:45,917 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:45,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3882d8b2ab81cfedea2b9eda817f13bf, ASSIGN because future has completed 2024-12-02T17:15:45,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1fd27a546540e9b8b8f7a078f3515500, ASSIGN because future has completed 2024-12-02T17:15:45,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fd27a546540e9b8b8f7a078f3515500, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:45,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0fd047fbb6ae4c8ccd6da6584c52f6db, ASSIGN because future has completed 2024-12-02T17:15:45,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fd047fbb6ae4c8ccd6da6584c52f6db, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:45,937 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/af5ee9a539ef801e1148eebca0987b9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:45,939 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened af5ee9a539ef801e1148eebca0987b9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69224410, jitterRate=0.031524091958999634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:45,939 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:45,940 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for af5ee9a539ef801e1148eebca0987b9c: Running coprocessor pre-open hook at 1733159745844Writing region info on filesystem at 1733159745844Initializing all the Stores at 1733159745847 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159745847Cleaning up temporary data from old regions at 1733159745883 (+36 ms)Running coprocessor post-open hooks at 1733159745939 (+56 ms)Region opened successfully at 1733159745940 (+1 ms) 2024-12-02T17:15:45,940 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:45,941 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 4eb95452ea2c6ad04c7114e1401424b9, NAME => 'test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-02T17:15:45,941 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,941 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:45,942 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,942 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,946 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c., pid=37, masterSystemTime=1733159745794 2024-12-02T17:15:45,954 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:45,954 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:45,954 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:45,955 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 31473ece20fde70fc9741ad3ff42aadc, NAME => 'test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-02T17:15:45,955 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:45,955 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:45,955 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:45,955 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:45,957 INFO [StoreOpener-4eb95452ea2c6ad04c7114e1401424b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,960 INFO [StoreOpener-4eb95452ea2c6ad04c7114e1401424b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4eb95452ea2c6ad04c7114e1401424b9 columnFamilyName family 2024-12-02T17:15:45,960 DEBUG [StoreOpener-4eb95452ea2c6ad04c7114e1401424b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:45,960 INFO [StoreOpener-31473ece20fde70fc9741ad3ff42aadc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:45,961 INFO [StoreOpener-4eb95452ea2c6ad04c7114e1401424b9-1 {}] regionserver.HStore(327): Store=4eb95452ea2c6ad04c7114e1401424b9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:45,961 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,962 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,963 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,964 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,964 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,968 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=af5ee9a539ef801e1148eebca0987b9c, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:45,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,39557,1733159739502 because future has completed 2024-12-02T17:15:45,974 INFO [StoreOpener-31473ece20fde70fc9741ad3ff42aadc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31473ece20fde70fc9741ad3ff42aadc columnFamilyName family 2024-12-02T17:15:45,974 DEBUG [StoreOpener-31473ece20fde70fc9741ad3ff42aadc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:45,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3882d8b2ab81cfedea2b9eda817f13bf, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:45,980 INFO [StoreOpener-31473ece20fde70fc9741ad3ff42aadc-1 {}] regionserver.HStore(327): Store=31473ece20fde70fc9741ad3ff42aadc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:45,981 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:45,983 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:45,985 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:45,987 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:45,987 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:45,993 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:45,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=22 2024-12-02T17:15:45,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,39557,1733159739502 in 216 msec 2024-12-02T17:15:45,998 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:46,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN in 574 msec 2024-12-02T17:15:46,001 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=6dd20102809dc64a92201c4ba1e20e38, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,002 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/4eb95452ea2c6ad04c7114e1401424b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,002 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=80be845edde76a33d61215cef1be07d7, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=eba2814ba0cf502df62df55d77023b23, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,003 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 4eb95452ea2c6ad04c7114e1401424b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58735970, jitterRate=-0.12476584315299988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,003 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:46,003 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 4eb95452ea2c6ad04c7114e1401424b9: Running coprocessor pre-open hook at 1733159745942Writing region info on filesystem at 1733159745942Initializing all the Stores at 1733159745956 (+14 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159745956Cleaning up temporary data from old regions at 1733159745964 (+8 ms)Running coprocessor post-open hooks at 1733159746003 (+39 ms)Region opened successfully at 1733159746003 2024-12-02T17:15:46,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6dd20102809dc64a92201c4ba1e20e38, ASSIGN because future has completed 2024-12-02T17:15:46,009 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/31473ece20fde70fc9741ad3ff42aadc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,009 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9., pid=39, masterSystemTime=1733159745857 2024-12-02T17:15:46,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=846f82385af02c9a3c05149f7ca40c3a, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,009 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 31473ece20fde70fc9741ad3ff42aadc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62980585, jitterRate=-0.06151615083217621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,010 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:46,010 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 31473ece20fde70fc9741ad3ff42aadc: Running coprocessor pre-open hook at 1733159745956Writing region info on filesystem at 1733159745956Initializing all the Stores at 1733159745957 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159745957Cleaning up temporary data from old regions at 1733159745987 (+30 ms)Running coprocessor post-open hooks at 1733159746010 (+23 ms)Region opened successfully at 1733159746010 2024-12-02T17:15:46,012 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc., pid=34, masterSystemTime=1733159745794 2024-12-02T17:15:46,013 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:46,013 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:46,013 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:46,013 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 51e940693c5f493c59a469d01db516d5, NAME => 'test,www,1733159743838.51e940693c5f493c59a469d01db516d5.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-02T17:15:46,014 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,014 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,www,1733159743838.51e940693c5f493c59a469d01db516d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,014 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,014 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,015 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:46,015 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:46,016 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:46,016 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => d53e8d53f3fcf0c3420445811190e425, NAME => 'test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-02T17:15:46,016 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,016 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,017 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,017 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=80be845edde76a33d61215cef1be07d7, ASSIGN because future has completed 2024-12-02T17:15:46,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eba2814ba0cf502df62df55d77023b23, ASSIGN because future has completed 2024-12-02T17:15:46,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=846f82385af02c9a3c05149f7ca40c3a, ASSIGN because future has completed 2024-12-02T17:15:46,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4eb95452ea2c6ad04c7114e1401424b9, server=3b3375d1a925,34079,1733159740078 because future has completed 2024-12-02T17:15:46,032 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T17:15:46,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=25 2024-12-02T17:15:46,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 4eb95452ea2c6ad04c7114e1401424b9, server=3b3375d1a925,34079,1733159740078 in 226 msec 2024-12-02T17:15:46,044 INFO [StoreOpener-51e940693c5f493c59a469d01db516d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4eb95452ea2c6ad04c7114e1401424b9, ASSIGN in 617 msec 2024-12-02T17:15:46,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 846f82385af02c9a3c05149f7ca40c3a, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:46,048 INFO [StoreOpener-d53e8d53f3fcf0c3420445811190e425-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:15:46,061 INFO [StoreOpener-d53e8d53f3fcf0c3420445811190e425-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d53e8d53f3fcf0c3420445811190e425 columnFamilyName family 2024-12-02T17:15:46,062 DEBUG [StoreOpener-d53e8d53f3fcf0c3420445811190e425-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure eba2814ba0cf502df62df55d77023b23, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:46,067 INFO [StoreOpener-51e940693c5f493c59a469d01db516d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51e940693c5f493c59a469d01db516d5 columnFamilyName family 2024-12-02T17:15:46,067 DEBUG [StoreOpener-51e940693c5f493c59a469d01db516d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80be845edde76a33d61215cef1be07d7, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:46,070 INFO [StoreOpener-d53e8d53f3fcf0c3420445811190e425-1 {}] regionserver.HStore(327): Store=d53e8d53f3fcf0c3420445811190e425/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,070 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,071 INFO [StoreOpener-51e940693c5f493c59a469d01db516d5-1 {}] regionserver.HStore(327): Store=51e940693c5f493c59a469d01db516d5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,071 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,072 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,072 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,073 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,074 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,074 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,074 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,074 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,075 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,077 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45949, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T17:15:46,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dd20102809dc64a92201c4ba1e20e38, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:46,081 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,099 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d53e8d53f3fcf0c3420445811190e425/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,106 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=31473ece20fde70fc9741ad3ff42aadc, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:46,112 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened d53e8d53f3fcf0c3420445811190e425; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59977870, jitterRate=-0.10626009106636047}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,113 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:46,113 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for d53e8d53f3fcf0c3420445811190e425: Running coprocessor pre-open hook at 1733159746017Writing region info on filesystem at 1733159746017Initializing all the Stores at 1733159746024 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746024Cleaning up temporary data from old regions at 1733159746074 (+50 ms)Running coprocessor post-open hooks at 1733159746113 (+39 ms)Region opened successfully at 1733159746113 2024-12-02T17:15:46,113 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/51e940693c5f493c59a469d01db516d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,114 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 51e940693c5f493c59a469d01db516d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59838542, jitterRate=-0.10833624005317688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,114 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=34, ppid=15, state=RUNNABLE, hasLock=true; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:46,114 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425., pid=31, masterSystemTime=1733159745794 2024-12-02T17:15:46,114 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:46,114 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 51e940693c5f493c59a469d01db516d5: Running coprocessor pre-open hook at 1733159746014Writing region info on filesystem at 1733159746014Initializing all the Stores at 1733159746024 (+10 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746024Cleaning up temporary data from old regions at 1733159746075 (+51 ms)Running coprocessor post-open hooks at 1733159746114 (+39 ms)Region opened successfully at 1733159746114 2024-12-02T17:15:46,119 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:46,119 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:46,119 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:46,119 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733159743838.51e940693c5f493c59a469d01db516d5., pid=33, masterSystemTime=1733159745857 2024-12-02T17:15:46,119 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => ba8c728daa66bac733cfa12bcd38a313, NAME => 'test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-02T17:15:46,120 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,120 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,120 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,120 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=15 2024-12-02T17:15:46,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,39557,1733159739502 in 405 msec 2024-12-02T17:15:46,124 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:46,124 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:46,124 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:46,124 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => e8a17d43fe45538f8e98a951b88f67de, NAME => 'test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-02T17:15:46,125 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,125 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,125 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,125 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN in 701 msec 2024-12-02T17:15:46,134 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=51e940693c5f493c59a469d01db516d5, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,137 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:46,137 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 3882d8b2ab81cfedea2b9eda817f13bf, NAME => 'test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-02T17:15:46,139 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,139 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,139 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,139 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,143 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=33, ppid=27, state=RUNNABLE, hasLock=true; OpenRegionProcedure 51e940693c5f493c59a469d01db516d5, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,153 INFO [StoreOpener-ba8c728daa66bac733cfa12bcd38a313-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,156 INFO [StoreOpener-ba8c728daa66bac733cfa12bcd38a313-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba8c728daa66bac733cfa12bcd38a313 columnFamilyName family 2024-12-02T17:15:46,157 DEBUG [StoreOpener-ba8c728daa66bac733cfa12bcd38a313-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=27 2024-12-02T17:15:46,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 51e940693c5f493c59a469d01db516d5, server=3b3375d1a925,34079,1733159740078 in 453 msec 2024-12-02T17:15:46,159 INFO [StoreOpener-ba8c728daa66bac733cfa12bcd38a313-1 {}] regionserver.HStore(327): Store=ba8c728daa66bac733cfa12bcd38a313/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,159 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,160 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,161 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,162 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,162 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,165 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=51e940693c5f493c59a469d01db516d5, ASSIGN in 735 msec 2024-12-02T17:15:46,169 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=d53e8d53f3fcf0c3420445811190e425, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:46,170 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,39557,1733159739502, table=test, region=d53e8d53f3fcf0c3420445811190e425. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:46,172 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:46,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure d53e8d53f3fcf0c3420445811190e425, server=3b3375d1a925,39557,1733159739502 because future has completed 2024-12-02T17:15:46,176 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN; state=OFFLINE, location=3b3375d1a925,39557,1733159739502; forceNewPlan=false, retain=false 2024-12-02T17:15:46,178 INFO [StoreOpener-e8a17d43fe45538f8e98a951b88f67de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-12-02T17:15:46,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure d53e8d53f3fcf0c3420445811190e425, server=3b3375d1a925,39557,1733159739502 in 573 msec 2024-12-02T17:15:46,186 INFO [StoreOpener-e8a17d43fe45538f8e98a951b88f67de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8a17d43fe45538f8e98a951b88f67de columnFamilyName family 2024-12-02T17:15:46,186 DEBUG [StoreOpener-e8a17d43fe45538f8e98a951b88f67de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN in 762 msec 2024-12-02T17:15:46,189 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee07072225557a5ffccbd5b4dc3f8250, ASSIGN 2024-12-02T17:15:46,190 INFO [StoreOpener-3882d8b2ab81cfedea2b9eda817f13bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,190 INFO [StoreOpener-e8a17d43fe45538f8e98a951b88f67de-1 {}] regionserver.HStore(327): Store=e8a17d43fe45538f8e98a951b88f67de/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,190 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,192 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee07072225557a5ffccbd5b4dc3f8250, ASSIGN; state=OFFLINE, location=3b3375d1a925,34079,1733159740078; forceNewPlan=false, retain=false 2024-12-02T17:15:46,193 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf8dc4ff69f7afd79a39737b7174af90, ASSIGN 2024-12-02T17:15:46,193 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,194 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,196 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bf8dc4ff69f7afd79a39737b7174af90, ASSIGN; state=OFFLINE, location=3b3375d1a925,33117,1733159739837; forceNewPlan=false, retain=false 2024-12-02T17:15:46,197 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,197 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,199 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,202 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc881fc21c14f92b22d6d3f233ac48b1, ASSIGN 2024-12-02T17:15:46,204 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/e8a17d43fe45538f8e98a951b88f67de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,206 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened e8a17d43fe45538f8e98a951b88f67de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71241414, jitterRate=0.06157979369163513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,206 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:46,207 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for e8a17d43fe45538f8e98a951b88f67de: Running coprocessor pre-open hook at 1733159746126Writing region info on filesystem at 1733159746126Initializing all the Stores at 1733159746128 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746128Cleaning up temporary data from old regions at 1733159746197 (+69 ms)Running coprocessor post-open hooks at 1733159746206 (+9 ms)Region opened successfully at 1733159746207 (+1 ms) 2024-12-02T17:15:46,207 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ba8c728daa66bac733cfa12bcd38a313/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,208 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fc881fc21c14f92b22d6d3f233ac48b1, ASSIGN; state=OFFLINE, location=3b3375d1a925,34079,1733159740078; forceNewPlan=false, retain=false 2024-12-02T17:15:46,208 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de., pid=43, masterSystemTime=1733159746006 2024-12-02T17:15:46,208 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened ba8c728daa66bac733cfa12bcd38a313; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59138210, jitterRate=-0.1187720000743866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,209 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:46,209 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for ba8c728daa66bac733cfa12bcd38a313: Running coprocessor pre-open hook at 1733159746120Writing region info on filesystem at 1733159746120Initializing all the Stores at 1733159746133 (+13 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746133Cleaning up temporary data from old regions at 1733159746162 (+29 ms)Running coprocessor post-open hooks at 1733159746209 (+47 ms)Region opened successfully at 1733159746209 2024-12-02T17:15:46,211 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313., pid=35, masterSystemTime=1733159745794 2024-12-02T17:15:46,211 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=16d4fe4d0593f960b6c0f8237bf0711b, ASSIGN 2024-12-02T17:15:46,212 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:46,212 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:46,212 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:46,212 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 8cd24aed41e647f8e01f1c4bdbf2c148, NAME => 'test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-02T17:15:46,212 INFO [StoreOpener-3882d8b2ab81cfedea2b9eda817f13bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3882d8b2ab81cfedea2b9eda817f13bf columnFamilyName family 2024-12-02T17:15:46,212 DEBUG [StoreOpener-3882d8b2ab81cfedea2b9eda817f13bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,213 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,213 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,214 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,214 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,215 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=16d4fe4d0593f960b6c0f8237bf0711b, ASSIGN; state=OFFLINE, location=3b3375d1a925,34079,1733159740078; forceNewPlan=false, retain=false 2024-12-02T17:15:46,216 INFO [StoreOpener-3882d8b2ab81cfedea2b9eda817f13bf-1 {}] regionserver.HStore(327): Store=3882d8b2ab81cfedea2b9eda817f13bf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,216 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,216 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:46,216 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:46,216 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:46,217 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => d612ab016a94fed4cbfb22a0c74d8f6a, NAME => 'test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-02T17:15:46,217 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,217 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,217 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,217 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,218 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,219 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,220 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,220 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,221 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ba8c728daa66bac733cfa12bcd38a313, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:46,225 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,39557,1733159739502 because future has completed 2024-12-02T17:15:46,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,235 INFO [StoreOpener-d612ab016a94fed4cbfb22a0c74d8f6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8a17d43fe45538f8e98a951b88f67de, server=3b3375d1a925,34079,1733159740078 because future has completed 2024-12-02T17:15:46,240 INFO [StoreOpener-d612ab016a94fed4cbfb22a0c74d8f6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d612ab016a94fed4cbfb22a0c74d8f6a columnFamilyName family 2024-12-02T17:15:46,240 DEBUG [StoreOpener-d612ab016a94fed4cbfb22a0c74d8f6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,241 INFO [StoreOpener-d612ab016a94fed4cbfb22a0c74d8f6a-1 {}] regionserver.HStore(327): Store=d612ab016a94fed4cbfb22a0c74d8f6a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,241 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,243 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,244 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,245 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,245 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=18 2024-12-02T17:15:46,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure e8a17d43fe45538f8e98a951b88f67de, server=3b3375d1a925,34079,1733159740078 in 342 msec 2024-12-02T17:15:46,246 INFO [StoreOpener-8cd24aed41e647f8e01f1c4bdbf2c148-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,248 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,248 INFO [StoreOpener-8cd24aed41e647f8e01f1c4bdbf2c148-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cd24aed41e647f8e01f1c4bdbf2c148 columnFamilyName family 2024-12-02T17:15:46,250 DEBUG [StoreOpener-8cd24aed41e647f8e01f1c4bdbf2c148-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,252 INFO [StoreOpener-8cd24aed41e647f8e01f1c4bdbf2c148-1 {}] regionserver.HStore(327): Store=8cd24aed41e647f8e01f1c4bdbf2c148/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,252 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,253 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,254 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e8a17d43fe45538f8e98a951b88f67de, ASSIGN in 823 msec 2024-12-02T17:15:46,254 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,254 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,258 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,260 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/3882d8b2ab81cfedea2b9eda817f13bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,261 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 3882d8b2ab81cfedea2b9eda817f13bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66629701, jitterRate=-0.0071400851011276245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,261 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:46,261 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 3882d8b2ab81cfedea2b9eda817f13bf: Running coprocessor pre-open hook at 1733159746139Writing region info on filesystem at 1733159746140 (+1 ms)Initializing all the Stores at 1733159746153 (+13 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746154 (+1 ms)Cleaning up temporary data from old regions at 1733159746220 (+66 ms)Running coprocessor post-open hooks at 1733159746261 (+41 ms)Region opened successfully at 1733159746261 2024-12-02T17:15:46,264 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf., pid=47, masterSystemTime=1733159746031 2024-12-02T17:15:46,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=17 2024-12-02T17:15:46,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,39557,1733159739502 in 529 msec 2024-12-02T17:15:46,266 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8cd24aed41e647f8e01f1c4bdbf2c148/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,267 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 8cd24aed41e647f8e01f1c4bdbf2c148; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69504884, jitterRate=0.03570348024368286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,267 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:46,267 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 8cd24aed41e647f8e01f1c4bdbf2c148: Running coprocessor pre-open hook at 1733159746215Writing region info on filesystem at 1733159746215Initializing all the Stores at 1733159746220 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746220Cleaning up temporary data from old regions at 1733159746254 (+34 ms)Running coprocessor post-open hooks at 1733159746267 (+13 ms)Region opened successfully at 1733159746267 2024-12-02T17:15:46,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN in 842 msec 2024-12-02T17:15:46,270 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148., pid=44, masterSystemTime=1733159746006 2024-12-02T17:15:46,271 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d612ab016a94fed4cbfb22a0c74d8f6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,273 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened d612ab016a94fed4cbfb22a0c74d8f6a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62482365, jitterRate=-0.06894020736217499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,274 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:46,274 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for d612ab016a94fed4cbfb22a0c74d8f6a: Running coprocessor pre-open hook at 1733159746217Writing region info on filesystem at 1733159746217Initializing all the Stores at 1733159746219 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746219Cleaning up temporary data from old regions at 1733159746245 (+26 ms)Running coprocessor post-open hooks at 1733159746274 (+29 ms)Region opened successfully at 1733159746274 2024-12-02T17:15:46,275 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a., pid=36, masterSystemTime=1733159745794 2024-12-02T17:15:46,275 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:46,276 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:46,276 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:46,276 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => cf990684b331d6bf1770501628bfa34f, NAME => 'test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-02T17:15:46,276 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,277 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,277 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,277 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,280 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:46,280 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:46,280 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:46,281 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fd27a546540e9b8b8f7a078f3515500, NAME => 'test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-02T17:15:46,281 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,281 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,281 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,281 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,283 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:46,283 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:46,284 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:46,284 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 76faba7cbca7c506fec4bb6f534477a2, NAME => 'test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-02T17:15:46,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,284 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,284 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,284 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,285 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cd24aed41e647f8e01f1c4bdbf2c148, server=3b3375d1a925,34079,1733159740078 because future has completed 2024-12-02T17:15:46,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:46,291 INFO [StoreOpener-cf990684b331d6bf1770501628bfa34f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,295 INFO [StoreOpener-cf990684b331d6bf1770501628bfa34f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf990684b331d6bf1770501628bfa34f columnFamilyName family 2024-12-02T17:15:46,295 DEBUG [StoreOpener-cf990684b331d6bf1770501628bfa34f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,296 INFO [StoreOpener-cf990684b331d6bf1770501628bfa34f-1 {}] regionserver.HStore(327): Store=cf990684b331d6bf1770501628bfa34f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,296 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,39557,1733159739502 because future has completed 2024-12-02T17:15:46,298 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,298 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,299 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,299 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,302 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,316 INFO [StoreOpener-1fd27a546540e9b8b8f7a078f3515500-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,328 INFO [3b3375d1a925:37351 {}] balancer.BaseLoadBalancer(388): Reassigned 5 regions. 5 retained the pre-restart assignment. 2024-12-02T17:15:46,334 INFO [StoreOpener-76faba7cbca7c506fec4bb6f534477a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,348 INFO [StoreOpener-1fd27a546540e9b8b8f7a078f3515500-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fd27a546540e9b8b8f7a078f3515500 columnFamilyName family 2024-12-02T17:15:46,348 DEBUG [StoreOpener-1fd27a546540e9b8b8f7a078f3515500-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,350 INFO [StoreOpener-1fd27a546540e9b8b8f7a078f3515500-1 {}] regionserver.HStore(327): Store=1fd27a546540e9b8b8f7a078f3515500/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,351 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,352 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,353 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,353 INFO [StoreOpener-76faba7cbca7c506fec4bb6f534477a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76faba7cbca7c506fec4bb6f534477a2 columnFamilyName family 2024-12-02T17:15:46,353 DEBUG [StoreOpener-76faba7cbca7c506fec4bb6f534477a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,353 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,353 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=24 2024-12-02T17:15:46,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 8cd24aed41e647f8e01f1c4bdbf2c148, server=3b3375d1a925,34079,1733159740078 in 389 msec 2024-12-02T17:15:46,356 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,357 INFO [StoreOpener-76faba7cbca7c506fec4bb6f534477a2-1 {}] regionserver.HStore(327): Store=76faba7cbca7c506fec4bb6f534477a2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,357 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,359 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,361 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/cf990684b331d6bf1770501628bfa34f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,361 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,362 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened cf990684b331d6bf1770501628bfa34f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65833926, jitterRate=-0.018998056650161743}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,362 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:46,362 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,362 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for cf990684b331d6bf1770501628bfa34f: Running coprocessor pre-open hook at 1733159746277Writing region info on filesystem at 1733159746277Initializing all the Stores at 1733159746281 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746281Cleaning up temporary data from old regions at 1733159746299 (+18 ms)Running coprocessor post-open hooks at 1733159746362 (+63 ms)Region opened successfully at 1733159746362 2024-12-02T17:15:46,362 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,364 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f., pid=42, masterSystemTime=1733159746006 2024-12-02T17:15:46,368 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8cd24aed41e647f8e01f1c4bdbf2c148, ASSIGN in 933 msec 2024-12-02T17:15:46,369 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:46,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,374 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,375 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,377 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:46,378 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:46,378 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:46,378 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 77b4e8908ad2baccd7eb5d5fa846dff8, NAME => 'test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-02T17:15:46,379 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,379 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,379 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,379 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,384 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,39557,1733159739502, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:46,385 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/1fd27a546540e9b8b8f7a078f3515500/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN because future has completed 2024-12-02T17:15:46,387 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 1fd27a546540e9b8b8f7a078f3515500; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66870480, jitterRate=-0.0035521984100341797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,387 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:46,388 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 1fd27a546540e9b8b8f7a078f3515500: Running coprocessor pre-open hook at 1733159746281Writing region info on filesystem at 1733159746281Initializing all the Stores at 1733159746284 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746284Cleaning up temporary data from old regions at 1733159746353 (+69 ms)Running coprocessor post-open hooks at 1733159746387 (+34 ms)Region opened successfully at 1733159746388 (+1 ms) 2024-12-02T17:15:46,398 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500., pid=45, masterSystemTime=1733159746031 2024-12-02T17:15:46,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee07072225557a5ffccbd5b4dc3f8250, ASSIGN because future has completed 2024-12-02T17:15:46,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fc881fc21c14f92b22d6d3f233ac48b1, ASSIGN because future has completed 2024-12-02T17:15:46,404 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/76faba7cbca7c506fec4bb6f534477a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,405 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 76faba7cbca7c506fec4bb6f534477a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72433394, jitterRate=0.07934167981147766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,406 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:46,406 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 76faba7cbca7c506fec4bb6f534477a2: Running coprocessor pre-open hook at 1733159746285Writing region info on filesystem at 1733159746285Initializing all the Stores at 1733159746287 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746287Cleaning up temporary data from old regions at 1733159746362 (+75 ms)Running coprocessor post-open hooks at 1733159746406 (+44 ms)Region opened successfully at 1733159746406 2024-12-02T17:15:46,409 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,409 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:46,409 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:46,409 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:46,410 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fd047fbb6ae4c8ccd6da6584c52f6db, NAME => 'test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-02T17:15:46,410 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,410 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,410 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,410 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,417 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2., pid=32, masterSystemTime=1733159745794 2024-12-02T17:15:46,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=16d4fe4d0593f960b6c0f8237bf0711b, ASSIGN because future has completed 2024-12-02T17:15:46,420 INFO [StoreOpener-77b4e8908ad2baccd7eb5d5fa846dff8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,424 INFO [StoreOpener-77b4e8908ad2baccd7eb5d5fa846dff8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77b4e8908ad2baccd7eb5d5fa846dff8 columnFamilyName family 2024-12-02T17:15:46,424 DEBUG [StoreOpener-77b4e8908ad2baccd7eb5d5fa846dff8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16d4fe4d0593f960b6c0f8237bf0711b, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:46,425 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:46,425 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:46,425 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:46,425 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => a37d5d00fe28247586762a18d6e0943f, NAME => 'test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-02T17:15:46,426 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,426 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,426 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,426 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,429 INFO [StoreOpener-77b4e8908ad2baccd7eb5d5fa846dff8-1 {}] regionserver.HStore(327): Store=77b4e8908ad2baccd7eb5d5fa846dff8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,429 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bf8dc4ff69f7afd79a39737b7174af90, ASSIGN because future has completed 2024-12-02T17:15:46,432 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,432 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,432 INFO [StoreOpener-0fd047fbb6ae4c8ccd6da6584c52f6db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf990684b331d6bf1770501628bfa34f, server=3b3375d1a925,34079,1733159740078 because future has completed 2024-12-02T17:15:46,434 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,434 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,436 INFO [StoreOpener-a37d5d00fe28247586762a18d6e0943f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,438 INFO [StoreOpener-0fd047fbb6ae4c8ccd6da6584c52f6db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fd047fbb6ae4c8ccd6da6584c52f6db columnFamilyName family 2024-12-02T17:15:46,438 DEBUG [StoreOpener-0fd047fbb6ae4c8ccd6da6584c52f6db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,438 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,439 INFO [StoreOpener-0fd047fbb6ae4c8ccd6da6584c52f6db-1 {}] regionserver.HStore(327): Store=0fd047fbb6ae4c8ccd6da6584c52f6db/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,439 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,440 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,441 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,441 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,442 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,441 INFO [StoreOpener-a37d5d00fe28247586762a18d6e0943f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a37d5d00fe28247586762a18d6e0943f columnFamilyName family 2024-12-02T17:15:46,445 DEBUG [StoreOpener-a37d5d00fe28247586762a18d6e0943f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,446 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,448 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/77b4e8908ad2baccd7eb5d5fa846dff8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,450 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 77b4e8908ad2baccd7eb5d5fa846dff8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67493664, jitterRate=0.005733966827392578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,450 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:46,450 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 77b4e8908ad2baccd7eb5d5fa846dff8: Running coprocessor pre-open hook at 1733159746379Writing region info on filesystem at 1733159746379Initializing all the Stores at 1733159746382 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746382Cleaning up temporary data from old regions at 1733159746434 (+52 ms)Running coprocessor post-open hooks at 1733159746450 (+16 ms)Region opened successfully at 1733159746450 2024-12-02T17:15:46,458 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/0fd047fbb6ae4c8ccd6da6584c52f6db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,459 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 0fd047fbb6ae4c8ccd6da6584c52f6db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69625665, jitterRate=0.037503257393836975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,459 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:46,459 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 0fd047fbb6ae4c8ccd6da6584c52f6db: Running coprocessor pre-open hook at 1733159746411Writing region info on filesystem at 1733159746411Initializing all the Stores at 1733159746412 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746412Cleaning up temporary data from old regions at 1733159746442 (+30 ms)Running coprocessor post-open hooks at 1733159746459 (+17 ms)Region opened successfully at 1733159746459 2024-12-02T17:15:46,459 INFO [StoreOpener-a37d5d00fe28247586762a18d6e0943f-1 {}] regionserver.HStore(327): Store=a37d5d00fe28247586762a18d6e0943f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,459 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,462 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,463 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,463 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,464 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,465 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8., pid=40, masterSystemTime=1733159746006 2024-12-02T17:15:46,466 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,469 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db., pid=46, masterSystemTime=1733159746031 2024-12-02T17:15:46,471 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/a37d5d00fe28247586762a18d6e0943f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,472 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened a37d5d00fe28247586762a18d6e0943f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72882513, jitterRate=0.08603407442569733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,472 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:46,472 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for a37d5d00fe28247586762a18d6e0943f: Running coprocessor pre-open hook at 1733159746428Writing region info on filesystem at 1733159746428Initializing all the Stores at 1733159746430 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746430Cleaning up temporary data from old regions at 1733159746464 (+34 ms)Running coprocessor post-open hooks at 1733159746472 (+8 ms)Region opened successfully at 1733159746472 2024-12-02T17:15:46,477 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f., pid=38, masterSystemTime=1733159745945 2024-12-02T17:15:46,487 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:46,487 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:46,496 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:46,496 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:46,496 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:46,497 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b72b8ea0bceb0151b39bea3aa7f2a9e, NAME => 'test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-02T17:15:46,497 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,497 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,497 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,497 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=12 2024-12-02T17:15:46,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure cf990684b331d6bf1770501628bfa34f, server=3b3375d1a925,34079,1733159740078 in 584 msec 2024-12-02T17:15:46,503 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:46,503 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:46,504 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,39557,1733159739502, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:46,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cf990684b331d6bf1770501628bfa34f, ASSIGN in 1.0760 sec 2024-12-02T17:15:46,509 INFO [StoreOpener-5b72b8ea0bceb0151b39bea3aa7f2a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,511 INFO [StoreOpener-5b72b8ea0bceb0151b39bea3aa7f2a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b72b8ea0bceb0151b39bea3aa7f2a9e columnFamilyName family 2024-12-02T17:15:46,511 DEBUG [StoreOpener-5b72b8ea0bceb0151b39bea3aa7f2a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a37d5d00fe28247586762a18d6e0943f, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:46,519 INFO [StoreOpener-5b72b8ea0bceb0151b39bea3aa7f2a9e-1 {}] regionserver.HStore(327): Store=5b72b8ea0bceb0151b39bea3aa7f2a9e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,519 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,520 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,521 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,522 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,522 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,524 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,540 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/5b72b8ea0bceb0151b39bea3aa7f2a9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,541 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 5b72b8ea0bceb0151b39bea3aa7f2a9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74778323, jitterRate=0.11428384482860565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,541 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:46,541 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 5b72b8ea0bceb0151b39bea3aa7f2a9e: Running coprocessor pre-open hook at 1733159746498Writing region info on filesystem at 1733159746498Initializing all the Stores at 1733159746501 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746501Cleaning up temporary data from old regions at 1733159746522 (+21 ms)Running coprocessor post-open hooks at 1733159746541 (+19 ms)Region opened successfully at 1733159746541 2024-12-02T17:15:46,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,39557,1733159739502 because future has completed 2024-12-02T17:15:46,547 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,550 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e., pid=41, masterSystemTime=1733159746031 2024-12-02T17:15:46,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fd047fbb6ae4c8ccd6da6584c52f6db, server=3b3375d1a925,33117,1733159739837 because future has completed 2024-12-02T17:15:46,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=23 2024-12-02T17:15:46,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,39557,1733159739502 in 765 msec 2024-12-02T17:15:46,560 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:46,560 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:46,560 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:46,560 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 846f82385af02c9a3c05149f7ca40c3a, NAME => 'test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-02T17:15:46,561 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,561 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,561 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,561 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,562 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,567 INFO [StoreOpener-846f82385af02c9a3c05149f7ca40c3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b72b8ea0bceb0151b39bea3aa7f2a9e, server=3b3375d1a925,33117,1733159739837 because future has completed 2024-12-02T17:15:46,575 INFO [StoreOpener-846f82385af02c9a3c05149f7ca40c3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 846f82385af02c9a3c05149f7ca40c3a columnFamilyName family 2024-12-02T17:15:46,575 DEBUG [StoreOpener-846f82385af02c9a3c05149f7ca40c3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=21 2024-12-02T17:15:46,580 INFO [StoreOpener-846f82385af02c9a3c05149f7ca40c3a-1 {}] regionserver.HStore(327): Store=846f82385af02c9a3c05149f7ca40c3a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fd047fbb6ae4c8ccd6da6584c52f6db, server=3b3375d1a925,33117,1733159739837 in 639 msec 2024-12-02T17:15:46,581 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,582 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,584 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,584 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,584 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0fd047fbb6ae4c8ccd6da6584c52f6db, ASSIGN in 1.1580 sec 2024-12-02T17:15:46,588 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,590 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:46,590 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 16d4fe4d0593f960b6c0f8237bf0711b, NAME => 'test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-02T17:15:46,590 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,590 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,591 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,591 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=30 2024-12-02T17:15:46,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b72b8ea0bceb0151b39bea3aa7f2a9e, server=3b3375d1a925,33117,1733159739837 in 712 msec 2024-12-02T17:15:46,596 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/846f82385af02c9a3c05149f7ca40c3a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,597 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 846f82385af02c9a3c05149f7ca40c3a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60737203, jitterRate=-0.09494514763355255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,597 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:46,598 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 846f82385af02c9a3c05149f7ca40c3a: Running coprocessor pre-open hook at 1733159746561Writing region info on filesystem at 1733159746561Initializing all the Stores at 1733159746563 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746563Cleaning up temporary data from old regions at 1733159746584 (+21 ms)Running coprocessor post-open hooks at 1733159746597 (+13 ms)Region opened successfully at 1733159746597 2024-12-02T17:15:46,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5b72b8ea0bceb0151b39bea3aa7f2a9e, ASSIGN in 1.1720 sec 2024-12-02T17:15:46,601 INFO [StoreOpener-16d4fe4d0593f960b6c0f8237bf0711b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN in 1.1750 sec 2024-12-02T17:15:46,611 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,39557,1733159739502, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:46,612 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a., pid=48, masterSystemTime=1733159746200 2024-12-02T17:15:46,616 INFO [StoreOpener-16d4fe4d0593f960b6c0f8237bf0711b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16d4fe4d0593f960b6c0f8237bf0711b columnFamilyName family 2024-12-02T17:15:46,616 DEBUG [StoreOpener-16d4fe4d0593f960b6c0f8237bf0711b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,616 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,616 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:46,616 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:46,617 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:46,617 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 80be845edde76a33d61215cef1be07d7, NAME => 'test,lll,1733159743838.80be845edde76a33d61215cef1be07d7.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-02T17:15:46,617 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,617 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,lll,1733159743838.80be845edde76a33d61215cef1be07d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,617 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,618 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,628 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,629 INFO [StoreOpener-16d4fe4d0593f960b6c0f8237bf0711b-1 {}] regionserver.HStore(327): Store=16d4fe4d0593f960b6c0f8237bf0711b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,629 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,630 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,632 INFO [StoreOpener-80be845edde76a33d61215cef1be07d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,637 INFO [StoreOpener-80be845edde76a33d61215cef1be07d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80be845edde76a33d61215cef1be07d7 columnFamilyName family 2024-12-02T17:15:46,637 DEBUG [StoreOpener-80be845edde76a33d61215cef1be07d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,638 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77b4e8908ad2baccd7eb5d5fa846dff8, server=3b3375d1a925,34079,1733159740078 because future has completed 2024-12-02T17:15:46,640 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,33117,1733159739837, table=test, region=846f82385af02c9a3c05149f7ca40c3a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:46,640 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,640 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 846f82385af02c9a3c05149f7ca40c3a, server=3b3375d1a925,33117,1733159739837 because future has completed 2024-12-02T17:15:46,643 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,644 INFO [StoreOpener-80be845edde76a33d61215cef1be07d7-1 {}] regionserver.HStore(327): Store=80be845edde76a33d61215cef1be07d7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,645 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=14 2024-12-02T17:15:46,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 77b4e8908ad2baccd7eb5d5fa846dff8, server=3b3375d1a925,34079,1733159740078 in 798 msec 2024-12-02T17:15:46,647 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,648 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,649 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,649 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,651 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=77b4e8908ad2baccd7eb5d5fa846dff8, ASSIGN in 1.2230 sec 2024-12-02T17:15:46,669 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/16d4fe4d0593f960b6c0f8237bf0711b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,670 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 16d4fe4d0593f960b6c0f8237bf0711b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62763751, jitterRate=-0.06474722921848297}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,670 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:46,670 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 16d4fe4d0593f960b6c0f8237bf0711b: Running coprocessor pre-open hook at 1733159746591Writing region info on filesystem at 1733159746591Initializing all the Stores at 1733159746592 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746592Cleaning up temporary data from old regions at 1733159746640 (+48 ms)Running coprocessor post-open hooks at 1733159746670 (+30 ms)Region opened successfully at 1733159746670 2024-12-02T17:15:46,674 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b., pid=52, masterSystemTime=1733159746583 2024-12-02T17:15:46,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=10 2024-12-02T17:15:46,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 846f82385af02c9a3c05149f7ca40c3a, server=3b3375d1a925,33117,1733159739837 in 620 msec 2024-12-02T17:15:46,689 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:46,689 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:46,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:46,695 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/80be845edde76a33d61215cef1be07d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,696 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 80be845edde76a33d61215cef1be07d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72606998, jitterRate=0.08192858099937439}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,696 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:46,696 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 80be845edde76a33d61215cef1be07d7: Running coprocessor pre-open hook at 1733159746618Writing region info on filesystem at 1733159746618Initializing all the Stores at 1733159746626 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746626Cleaning up temporary data from old regions at 1733159746649 (+23 ms)Running coprocessor post-open hooks at 1733159746696 (+47 ms)Region opened successfully at 1733159746696 2024-12-02T17:15:46,698 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733159743838.80be845edde76a33d61215cef1be07d7., pid=50, masterSystemTime=1733159746200 2024-12-02T17:15:46,702 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:46,702 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:46,702 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:46,703 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => eba2814ba0cf502df62df55d77023b23, NAME => 'test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-02T17:15:46,703 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,703 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,703 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,703 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=846f82385af02c9a3c05149f7ca40c3a, ASSIGN in 1.2760 sec 2024-12-02T17:15:46,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16d4fe4d0593f960b6c0f8237bf0711b, server=3b3375d1a925,34079,1733159740078 because future has completed 2024-12-02T17:15:46,706 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=80be845edde76a33d61215cef1be07d7, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80be845edde76a33d61215cef1be07d7, server=3b3375d1a925,33117,1733159739837 because future has completed 2024-12-02T17:15:46,715 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,39557,1733159739502, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:46,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=5 2024-12-02T17:15:46,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 16d4fe4d0593f960b6c0f8237bf0711b, server=3b3375d1a925,34079,1733159740078 in 286 msec 2024-12-02T17:15:46,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=16d4fe4d0593f960b6c0f8237bf0711b, ASSIGN in 1.2970 sec 2024-12-02T17:15:46,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=16 2024-12-02T17:15:46,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 80be845edde76a33d61215cef1be07d7, server=3b3375d1a925,33117,1733159739837 in 656 msec 2024-12-02T17:15:46,730 INFO [StoreOpener-eba2814ba0cf502df62df55d77023b23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,734 INFO [StoreOpener-eba2814ba0cf502df62df55d77023b23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eba2814ba0cf502df62df55d77023b23 columnFamilyName family 2024-12-02T17:15:46,734 DEBUG [StoreOpener-eba2814ba0cf502df62df55d77023b23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=80be845edde76a33d61215cef1be07d7, ASSIGN in 1.3070 sec 2024-12-02T17:15:46,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf8dc4ff69f7afd79a39737b7174af90, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:46,738 INFO [StoreOpener-eba2814ba0cf502df62df55d77023b23-1 {}] regionserver.HStore(327): Store=eba2814ba0cf502df62df55d77023b23/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,738 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,739 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,740 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,740 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,740 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,743 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,756 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=76faba7cbca7c506fec4bb6f534477a2, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:46,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,39557,1733159739502 because future has completed 2024-12-02T17:15:46,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fd27a546540e9b8b8f7a078f3515500, server=3b3375d1a925,33117,1733159739837 because future has completed 2024-12-02T17:15:46,770 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/eba2814ba0cf502df62df55d77023b23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,771 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened eba2814ba0cf502df62df55d77023b23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73934402, jitterRate=0.10170844197273254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,772 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:46,772 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for eba2814ba0cf502df62df55d77023b23: Running coprocessor pre-open hook at 1733159746703Writing region info on filesystem at 1733159746703Initializing all the Stores at 1733159746709 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746709Cleaning up temporary data from old regions at 1733159746740 (+31 ms)Running coprocessor post-open hooks at 1733159746772 (+32 ms)Region opened successfully at 1733159746772 2024-12-02T17:15:46,785 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23., pid=49, masterSystemTime=1733159746200 2024-12-02T17:15:46,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=26 2024-12-02T17:15:46,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 1fd27a546540e9b8b8f7a078f3515500, server=3b3375d1a925,33117,1733159739837 in 853 msec 2024-12-02T17:15:46,792 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:46,792 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:46,793 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:46,793 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 6dd20102809dc64a92201c4ba1e20e38, NAME => 'test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-02T17:15:46,793 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,793 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,794 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,794 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,797 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,805 INFO [StoreOpener-6dd20102809dc64a92201c4ba1e20e38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure eba2814ba0cf502df62df55d77023b23, server=3b3375d1a925,33117,1733159739837 because future has completed 2024-12-02T17:15:46,813 INFO [StoreOpener-6dd20102809dc64a92201c4ba1e20e38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6dd20102809dc64a92201c4ba1e20e38 columnFamilyName family 2024-12-02T17:15:46,813 DEBUG [StoreOpener-6dd20102809dc64a92201c4ba1e20e38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1fd27a546540e9b8b8f7a078f3515500, ASSIGN in 1.3820 sec 2024-12-02T17:15:46,817 INFO [StoreOpener-6dd20102809dc64a92201c4ba1e20e38-1 {}] regionserver.HStore(327): Store=6dd20102809dc64a92201c4ba1e20e38/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,818 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,819 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,819 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,820 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,820 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,823 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,826 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,39557,1733159739502, table=test, region=76faba7cbca7c506fec4bb6f534477a2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:46,826 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,39557,1733159739502, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:46,836 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/6dd20102809dc64a92201c4ba1e20e38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=13 2024-12-02T17:15:46,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure eba2814ba0cf502df62df55d77023b23, server=3b3375d1a925,33117,1733159739837 in 763 msec 2024-12-02T17:15:46,837 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 6dd20102809dc64a92201c4ba1e20e38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65981006, jitterRate=-0.01680639386177063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,838 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:46,838 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 6dd20102809dc64a92201c4ba1e20e38: Running coprocessor pre-open hook at 1733159746794Writing region info on filesystem at 1733159746794Initializing all the Stores at 1733159746798 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746798Cleaning up temporary data from old regions at 1733159746820 (+22 ms)Running coprocessor post-open hooks at 1733159746838 (+18 ms)Region opened successfully at 1733159746838 2024-12-02T17:15:46,845 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38., pid=51, masterSystemTime=1733159746200 2024-12-02T17:15:46,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eba2814ba0cf502df62df55d77023b23, ASSIGN in 1.4140 sec 2024-12-02T17:15:46,855 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:46,855 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:46,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=11 2024-12-02T17:15:46,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,39557,1733159739502 in 1.1840 sec 2024-12-02T17:15:46,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN in 1.4430 sec 2024-12-02T17:15:46,870 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc881fc21c14f92b22d6d3f233ac48b1, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:46,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dd20102809dc64a92201c4ba1e20e38, server=3b3375d1a925,33117,1733159739837 because future has completed 2024-12-02T17:15:46,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=19 2024-12-02T17:15:46,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 6dd20102809dc64a92201c4ba1e20e38, server=3b3375d1a925,33117,1733159739837 in 803 msec 2024-12-02T17:15:46,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6dd20102809dc64a92201c4ba1e20e38, ASSIGN in 1.4630 sec 2024-12-02T17:15:46,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee07072225557a5ffccbd5b4dc3f8250, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:46,907 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:46,908 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => bf8dc4ff69f7afd79a39737b7174af90, NAME => 'test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-02T17:15:46,908 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,908 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:46,908 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,908 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,917 INFO [StoreOpener-bf8dc4ff69f7afd79a39737b7174af90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,39557,1733159739502}] 2024-12-02T17:15:46,931 INFO [StoreOpener-bf8dc4ff69f7afd79a39737b7174af90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf8dc4ff69f7afd79a39737b7174af90 columnFamilyName family 2024-12-02T17:15:46,931 DEBUG [StoreOpener-bf8dc4ff69f7afd79a39737b7174af90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:46,932 INFO [StoreOpener-bf8dc4ff69f7afd79a39737b7174af90-1 {}] regionserver.HStore(327): Store=bf8dc4ff69f7afd79a39737b7174af90/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:46,932 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,935 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,936 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,39557,1733159739502, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:46,936 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,937 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,937 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,944 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=20 2024-12-02T17:15:46,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,39557,1733159739502 in 1.2150 sec 2024-12-02T17:15:46,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN in 1.5390 sec 2024-12-02T17:15:46,974 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/bf8dc4ff69f7afd79a39737b7174af90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:46,975 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened bf8dc4ff69f7afd79a39737b7174af90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66186578, jitterRate=-0.01374313235282898}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:46,975 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:46,975 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:46,975 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for bf8dc4ff69f7afd79a39737b7174af90: Running coprocessor pre-open hook at 1733159746909Writing region info on filesystem at 1733159746909Initializing all the Stores at 1733159746910 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159746910Cleaning up temporary data from old regions at 1733159746937 (+27 ms)Running coprocessor post-open hooks at 1733159746975 (+38 ms)Region opened successfully at 1733159746975 2024-12-02T17:15:46,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3882d8b2ab81cfedea2b9eda817f13bf, server=3b3375d1a925,33117,1733159739837 because future has completed 2024-12-02T17:15:46,981 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90., pid=53, masterSystemTime=1733159746901 2024-12-02T17:15:46,993 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:46,994 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:47,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=29 2024-12-02T17:15:47,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 3882d8b2ab81cfedea2b9eda817f13bf, server=3b3375d1a925,33117,1733159739837 in 1.0110 sec 2024-12-02T17:15:47,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3882d8b2ab81cfedea2b9eda817f13bf, ASSIGN in 1.5880 sec 2024-12-02T17:15:47,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:47,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf8dc4ff69f7afd79a39737b7174af90, server=3b3375d1a925,33117,1733159739837 because future has completed 2024-12-02T17:15:47,045 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:47,045 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => ee07072225557a5ffccbd5b4dc3f8250, NAME => 'test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-02T17:15:47,045 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,045 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:47,046 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,046 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-12-02T17:15:47,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure bf8dc4ff69f7afd79a39737b7174af90, server=3b3375d1a925,33117,1733159739837 in 321 msec 2024-12-02T17:15:47,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bf8dc4ff69f7afd79a39737b7174af90, ASSIGN in 1.6440 sec 2024-12-02T17:15:47,085 INFO [StoreOpener-ee07072225557a5ffccbd5b4dc3f8250-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,097 INFO [StoreOpener-ee07072225557a5ffccbd5b4dc3f8250-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee07072225557a5ffccbd5b4dc3f8250 columnFamilyName family 2024-12-02T17:15:47,097 DEBUG [StoreOpener-ee07072225557a5ffccbd5b4dc3f8250-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:47,098 INFO [StoreOpener-ee07072225557a5ffccbd5b4dc3f8250-1 {}] regionserver.HStore(327): Store=ee07072225557a5ffccbd5b4dc3f8250/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:47,099 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,100 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,100 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,101 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,101 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,103 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,103 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:47,104 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 8981ccd8f09f082161872d7b109756f0, NAME => 'test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-02T17:15:47,104 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,104 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:47,105 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,105 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,128 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ee07072225557a5ffccbd5b4dc3f8250/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:47,129 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened ee07072225557a5ffccbd5b4dc3f8250; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72225130, jitterRate=0.07623830437660217}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:47,130 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:47,130 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for ee07072225557a5ffccbd5b4dc3f8250: Running coprocessor pre-open hook at 1733159747046Writing region info on filesystem at 1733159747046Initializing all the Stores at 1733159747047 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159747047Cleaning up temporary data from old regions at 1733159747101 (+54 ms)Running coprocessor post-open hooks at 1733159747130 (+29 ms)Region opened successfully at 1733159747130 2024-12-02T17:15:47,132 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250., pid=55, masterSystemTime=1733159747032 2024-12-02T17:15:47,140 INFO [StoreOpener-8981ccd8f09f082161872d7b109756f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,140 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:47,140 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:47,141 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:47,141 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => fc881fc21c14f92b22d6d3f233ac48b1, NAME => 'test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-02T17:15:47,141 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,141 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:47,141 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,142 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:47,146 INFO [StoreOpener-8981ccd8f09f082161872d7b109756f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8981ccd8f09f082161872d7b109756f0 columnFamilyName family 2024-12-02T17:15:47,147 DEBUG [StoreOpener-8981ccd8f09f082161872d7b109756f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:47,147 INFO [StoreOpener-8981ccd8f09f082161872d7b109756f0-1 {}] regionserver.HStore(327): Store=8981ccd8f09f082161872d7b109756f0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:47,148 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,149 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,149 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,150 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,150 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,152 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,167 INFO [StoreOpener-fc881fc21c14f92b22d6d3f233ac48b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,172 INFO [StoreOpener-fc881fc21c14f92b22d6d3f233ac48b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc881fc21c14f92b22d6d3f233ac48b1 columnFamilyName family 2024-12-02T17:15:47,173 DEBUG [StoreOpener-fc881fc21c14f92b22d6d3f233ac48b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:47,173 INFO [StoreOpener-fc881fc21c14f92b22d6d3f233ac48b1-1 {}] regionserver.HStore(327): Store=fc881fc21c14f92b22d6d3f233ac48b1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:47,173 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,174 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,175 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,176 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,176 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee07072225557a5ffccbd5b4dc3f8250, server=3b3375d1a925,34079,1733159740078 because future has completed 2024-12-02T17:15:47,180 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,190 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8981ccd8f09f082161872d7b109756f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:47,191 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 8981ccd8f09f082161872d7b109756f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71949599, jitterRate=0.07213257253170013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:47,191 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:47,191 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 8981ccd8f09f082161872d7b109756f0: Running coprocessor pre-open hook at 1733159747105Writing region info on filesystem at 1733159747105Initializing all the Stores at 1733159747108 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159747108Cleaning up temporary data from old regions at 1733159747150 (+42 ms)Running coprocessor post-open hooks at 1733159747191 (+41 ms)Region opened successfully at 1733159747191 2024-12-02T17:15:47,193 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0., pid=56, masterSystemTime=1733159747096 2024-12-02T17:15:47,196 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/fc881fc21c14f92b22d6d3f233ac48b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:47,197 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened fc881fc21c14f92b22d6d3f233ac48b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73819012, jitterRate=0.09998899698257446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:47,197 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:47,197 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for fc881fc21c14f92b22d6d3f233ac48b1: Running coprocessor pre-open hook at 1733159747142Writing region info on filesystem at 1733159747142Initializing all the Stores at 1733159747145 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159747145Cleaning up temporary data from old regions at 1733159747176 (+31 ms)Running coprocessor post-open hooks at 1733159747197 (+21 ms)Region opened successfully at 1733159747197 2024-12-02T17:15:47,202 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:47,202 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:47,202 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1., pid=54, masterSystemTime=1733159747032 2024-12-02T17:15:47,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=8 2024-12-02T17:15:47,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ee07072225557a5ffccbd5b4dc3f8250, server=3b3375d1a925,34079,1733159740078 in 295 msec 2024-12-02T17:15:47,206 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:47,206 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:47,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ee07072225557a5ffccbd5b4dc3f8250, ASSIGN in 1.7820 sec 2024-12-02T17:15:47,211 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:47,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc881fc21c14f92b22d6d3f233ac48b1, server=3b3375d1a925,34079,1733159740078 because future has completed 2024-12-02T17:15:47,222 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8981ccd8f09f082161872d7b109756f0, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,39557,1733159739502 2024-12-02T17:15:47,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,39557,1733159739502 because future has completed 2024-12-02T17:15:47,255 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3b3375d1a925,39557,1733159739502, table=test, region=8981ccd8f09f082161872d7b109756f0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T17:15:47,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=6 2024-12-02T17:15:47,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure fc881fc21c14f92b22d6d3f233ac48b1, server=3b3375d1a925,34079,1733159740078 in 368 msec 2024-12-02T17:15:47,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fc881fc21c14f92b22d6d3f233ac48b1, ASSIGN in 1.8500 sec 2024-12-02T17:15:47,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=9 2024-12-02T17:15:47,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,39557,1733159739502 in 361 msec 2024-12-02T17:15:47,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=4 2024-12-02T17:15:47,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN in 1.8900 sec 2024-12-02T17:15:47,320 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T17:15:47,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159747320"}]},"ts":"1733159747320"} 2024-12-02T17:15:47,344 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-02T17:15:47,348 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T17:15:47,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 3.5000 sec 2024-12-02T17:15:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:15:48,081 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-02T17:15:48,081 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-02T17:15:48,084 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T17:15:48,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-02T17:15:48,149 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T17:15:48,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-02T17:15:48,150 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T17:15:48,177 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-02T17:15:48,225 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-02T17:15:48,265 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test', row='', locateType=CURRENT is [region=test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b., hostname=3b3375d1a925,34079,1733159740078, seqNum=2] 2024-12-02T17:15:48,272 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:48,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:48,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:48,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bf8dc4ff69f7afd79a39737b7174af90 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 846f82385af02c9a3c05149f7ca40c3a changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eba2814ba0cf502df62df55d77023b23 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 80be845edde76a33d61215cef1be07d7 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6dd20102809dc64a92201c4ba1e20e38 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0fd047fbb6ae4c8ccd6da6584c52f6db changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1fd27a546540e9b8b8f7a078f3515500 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3882d8b2ab81cfedea2b9eda817f13bf changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5b72b8ea0bceb0151b39bea3aa7f2a9e changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 16d4fe4d0593f960b6c0f8237bf0711b changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fc881fc21c14f92b22d6d3f233ac48b1 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ee07072225557a5ffccbd5b4dc3f8250 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cf990684b331d6bf1770501628bfa34f changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 77b4e8908ad2baccd7eb5d5fa846dff8 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e8a17d43fe45538f8e98a951b88f67de changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8cd24aed41e647f8e01f1c4bdbf2c148 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4eb95452ea2c6ad04c7114e1401424b9 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 51e940693c5f493c59a469d01db516d5 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8981ccd8f09f082161872d7b109756f0 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 76faba7cbca7c506fec4bb6f534477a2 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 31473ece20fde70fc9741ad3ff42aadc changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ba8c728daa66bac733cfa12bcd38a313 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d612ab016a94fed4cbfb22a0c74d8f6a changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region af5ee9a539ef801e1148eebca0987b9c changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a37d5d00fe28247586762a18d6e0943f changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d53e8d53f3fcf0c3420445811190e425 changed from -1.0 to 0.0, refreshing cache 2024-12-02T17:15:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-02T17:15:48,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-02T17:15:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-02T17:15:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(204): Hosts are {3b3375d1a925=0} racks are {/default-rack=0} 2024-12-02T17:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-12-02T17:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-12-02T17:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-12-02T17:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T17:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T17:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T17:15:48,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T17:15:48,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T17:15:48,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T17:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-02T17:15:48,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-02T17:15:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-02T17:15:48,499 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T17:15:48,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-12-02T17:15:48,567 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=378, OpenFileDescriptor=647, MaxFileDescriptor=1048576, SystemLoadAverage=1043, ProcessCount=11, AvailableMemoryMB=3575 2024-12-02T17:15:48,573 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-02T17:15:48,574 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-02T17:15:48,574 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 3b3375d1a925,39557,1733159739502 2024-12-02T17:15:48,576 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 3b3375d1a925,39557,1733159739502: Simulated kill ***** 2024-12-02T17:15:48,576 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-02T17:15:48,583 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-02T17:15:48,586 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-02T17:15:48,586 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-02T17:15:48,586 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-02T17:15:48,588 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 329370720 }, "NonHeapMemoryUsage": { "committed": 152895488, "init": 7667712, "max": -1, "used": 150641056 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-12-02T17:15:48,592 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37351 {}] master.MasterRpcServices(700): 3b3375d1a925,39557,1733159739502 reported a fatal error: ***** ABORTING region server 3b3375d1a925,39557,1733159739502: Simulated kill ***** 2024-12-02T17:15:48,597 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3b3375d1a925,39557,1733159739502' ***** 2024-12-02T17:15:48,597 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-02T17:15:48,597 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T17:15:48,597 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T17:15:48,598 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T17:15:48,605 INFO [RS:0;3b3375d1a925:39557 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-02T17:15:48,605 INFO [RS:0;3b3375d1a925:39557 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-02T17:15:48,605 INFO [RS:0;3b3375d1a925:39557 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T17:15:48,605 INFO [RS:0;3b3375d1a925:39557 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3b3375d1a925:39557. 2024-12-02T17:15:48,605 DEBUG [RS:0;3b3375d1a925:39557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:15:48,614 DEBUG [RS:0;3b3375d1a925:39557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:48,617 INFO [regionserver/3b3375d1a925:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T17:15:48,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741836_1012 (size=32114) 2024-12-02T17:15:48,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741836_1012 (size=32114) 2024-12-02T17:15:48,666 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T17:15:48,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741836_1012 (size=32114) 2024-12-02T17:15:48,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741835_1011 (size=2237) 2024-12-02T17:15:48,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741835_1011 (size=2237) 2024-12-02T17:15:48,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741835_1011 (size=2237) 2024-12-02T17:15:48,701 DEBUG [RS:0;3b3375d1a925:39557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:48,701 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T17:15:48,701 INFO [RS:0;3b3375d1a925:39557 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39557 2024-12-02T17:15:48,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T17:15:48,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3b3375d1a925,39557,1733159739502 2024-12-02T17:15:48,730 INFO [RS:0;3b3375d1a925:39557 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T17:15:48,738 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3b3375d1a925,39557,1733159739502] 2024-12-02T17:15:48,756 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3b3375d1a925,39557,1733159739502 already deleted, retry=false 2024-12-02T17:15:48,757 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 3b3375d1a925,39557,1733159739502 on 3b3375d1a925,37351,1733159738385 2024-12-02T17:15:48,769 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:48,778 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 3b3375d1a925,39557,1733159739502 (carryingMeta=true) 3b3375d1a925,39557,1733159739502/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5b3c4c7f[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-02T17:15:48,778 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:48,780 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true, isMeta: true 2024-12-02T17:15:48,784 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting 2024-12-02T17:15:48,786 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3b3375d1a925,39557,1733159739502 WAL count=1, meta=true 2024-12-02T17:15:48,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta}] 2024-12-02T17:15:48,798 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:48,802 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T17:15:48,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:48,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:15:48,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x1009bfce1af0001, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:15:48,837 INFO [RS:0;3b3375d1a925:39557 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T17:15:48,838 INFO [RS:0;3b3375d1a925:39557 {}] regionserver.HRegionServer(1031): Exiting; stopping=3b3375d1a925,39557,1733159739502; zookeeper connection closed. 2024-12-02T17:15:48,846 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@799807a7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@799807a7 2024-12-02T17:15:48,850 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T17:15:48,866 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-02T17:15:48,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T17:15:48,968 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T17:15:48,977 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-12-02T17:15:48,977 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-12-02T17:15:48,978 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T17:15:48,978 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T17:15:48,979 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T17:15:48,979 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T17:15:48,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-12-02T17:15:49,039 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, size=31.4 K (32114bytes) 2024-12-02T17:15:49,040 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-02T17:15:49,046 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-02T17:15:49,047 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta 2024-12-02T17:15:49,060 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta after 10ms 2024-12-02T17:15:49,083 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta took 45ms 2024-12-02T17:15:49,114 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-02T17:15:49,130 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta.temp 2024-12-02T17:15:49,142 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta so closing down 2024-12-02T17:15:49,142 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T17:15:49,149 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta.temp 2024-12-02T17:15:49,154 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T17:15:49,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741864_1040 (size=31873) 2024-12-02T17:15:49,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741864_1040 (size=31873) 2024-12-02T17:15:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741864_1040 (size=31873) 2024-12-02T17:15:49,180 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-12-02T17:15:49,183 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta.temp to hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-02T17:15:49,196 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 88 ms; skipped=1; WAL=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-12-02T17:15:49,196 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, journal: Splitting hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, size=31.4 K (32114bytes) at 1733159749039Finishing writing output for hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta so closing down at 1733159749142 (+103 ms)Creating recovered edits writer path=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta.temp at 1733159749149 (+7 ms)3 split writer threads finished at 1733159749154 (+5 ms)Closed recovered edits writer path=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1733159749180 (+26 ms)Rename recovered edits hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta.temp to hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1733159749183 (+3 ms)Processed 58 edits across 1 Regions in 88 ms; skipped=1; WAL=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1733159749196 (+13 ms) 2024-12-02T17:15:49,196 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta 2024-12-02T17:15:49,202 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-02T17:15:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-02T17:15:49,218 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta to hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs 2024-12-02T17:15:49,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-02T17:15:49,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 in 415 msec 2024-12-02T17:15:49,267 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:49,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-02T17:15:49,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 in 479 msec 2024-12-02T17:15:49,278 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting dir is empty, no logs to split. 2024-12-02T17:15:49,278 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3b3375d1a925,39557,1733159739502 WAL count=0, meta=true 2024-12-02T17:15:49,278 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3b3375d1a925,39557,1733159739502 WAL splitting is done? wals=0, meta=true 2024-12-02T17:15:49,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T17:15:49,285 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:49,287 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T17:15:49,330 INFO [regionserver/3b3375d1a925:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T17:15:49,330 INFO [regionserver/3b3375d1a925:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T17:15:49,452 DEBUG [3b3375d1a925:37351 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-02T17:15:49,452 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(204): Hosts are {3b3375d1a925=0} racks are {/default-rack=0} 2024-12-02T17:15:49,453 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T17:15:49,453 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T17:15:49,453 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T17:15:49,453 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T17:15:49,453 INFO [3b3375d1a925:37351 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T17:15:49,453 INFO [3b3375d1a925:37351 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T17:15:49,453 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T17:15:49,453 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:49,461 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPENING 2024-12-02T17:15:49,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:49,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:49,464 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:49,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:49,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:49,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:49,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:49,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:49,681 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T17:15:49,682 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T17:15:49,682 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T17:15:49,706 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b3375d1a925%2C33117%2C1733159739837.meta, suffix=.meta, logDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,33117,1733159739837, archiveDir=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs, maxLogs=32 2024-12-02T17:15:49,755 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,33117,1733159739837/3b3375d1a925%2C33117%2C1733159739837.meta.1733159749707.meta, exclude list is [], retry=0 2024-12-02T17:15:49,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40475,DS-dd8dc82b-5639-4510-9136-c7aeaf6c3351,DISK] 2024-12-02T17:15:49,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38149,DS-668e2095-1fca-40fa-bb76-12a999cf7985,DISK] 2024-12-02T17:15:49,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41165,DS-a26ba80d-4012-4587-b072-51db620caa12,DISK] 2024-12-02T17:15:49,813 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,33117,1733159739837/3b3375d1a925%2C33117%2C1733159739837.meta.1733159749707.meta 2024-12-02T17:15:49,825 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37663:37663),(127.0.0.1/127.0.0.1:46411:46411),(127.0.0.1/127.0.0.1:44061:44061)] 2024-12-02T17:15:49,825 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:15:49,825 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T17:15:49,826 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T17:15:49,826 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T17:15:49,826 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T17:15:49,826 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:49,826 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T17:15:49,826 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T17:15:49,834 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T17:15:49,835 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T17:15:49,835 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:49,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:49,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T17:15:49,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T17:15:49,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:49,850 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:49,850 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T17:15:49,852 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T17:15:49,852 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:49,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:49,854 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T17:15:49,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T17:15:49,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:49,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:15:49,859 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T17:15:49,860 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740 2024-12-02T17:15:49,864 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740 2024-12-02T17:15:49,865 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-02T17:15:49,903 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-02T17:15:49,914 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-12-02T17:15:50,120 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/info/f6983332803844fb9990c23850a5f82b is 138, key is test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1./info:regioninfo/1733159747211/Put/seqid=0 2024-12-02T17:15:50,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741866_1042 (size=34924) 2024-12-02T17:15:50,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741866_1042 (size=34924) 2024-12-02T17:15:50,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741866_1042 (size=34924) 2024-12-02T17:15:50,221 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/info/f6983332803844fb9990c23850a5f82b 2024-12-02T17:15:50,381 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/ns/11c80017c91e4d94ac251d88bbacbf65 is 43, key is default/ns:d/1733159743367/Put/seqid=0 2024-12-02T17:15:50,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741867_1043 (size=5153) 2024-12-02T17:15:50,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741867_1043 (size=5153) 2024-12-02T17:15:50,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741867_1043 (size=5153) 2024-12-02T17:15:50,475 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/ns/11c80017c91e4d94ac251d88bbacbf65 2024-12-02T17:15:50,567 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/table/8fd6cac9a6414f279cae5942e8cd60ee is 40, key is test/table:state/1733159747320/Put/seqid=0 2024-12-02T17:15:50,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741868_1044 (size=5165) 2024-12-02T17:15:50,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741868_1044 (size=5165) 2024-12-02T17:15:50,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741868_1044 (size=5165) 2024-12-02T17:15:50,633 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/table/8fd6cac9a6414f279cae5942e8cd60ee 2024-12-02T17:15:50,666 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/info/f6983332803844fb9990c23850a5f82b as hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/info/f6983332803844fb9990c23850a5f82b 2024-12-02T17:15:50,679 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/info/f6983332803844fb9990c23850a5f82b, entries=260, sequenceid=60, filesize=34.1 K 2024-12-02T17:15:50,691 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/ns/11c80017c91e4d94ac251d88bbacbf65 as hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/ns/11c80017c91e4d94ac251d88bbacbf65 2024-12-02T17:15:50,702 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/ns/11c80017c91e4d94ac251d88bbacbf65, entries=2, sequenceid=60, filesize=5.0 K 2024-12-02T17:15:50,708 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/table/8fd6cac9a6414f279cae5942e8cd60ee as hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/table/8fd6cac9a6414f279cae5942e8cd60ee 2024-12-02T17:15:50,723 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/table/8fd6cac9a6414f279cae5942e8cd60ee, entries=2, sequenceid=60, filesize=5.0 K 2024-12-02T17:15:50,735 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 822ms, sequenceid=60, compaction requested=false; wal=null 2024-12-02T17:15:50,737 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-02T17:15:50,742 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T17:15:50,742 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T17:15:50,743 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T17:15:50,748 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T17:15:50,764 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-12-02T17:15:50,767 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68182391, jitterRate=0.015996798872947693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T17:15:50,767 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T17:15:50,767 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733159749826Writing region info on filesystem at 1733159749826Initializing all the Stores at 1733159749828 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159749828Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159749833 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159749833Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159749833Obtaining lock to block concurrent updates at 1733159749914 (+81 ms)Preparing flush snapshotting stores in 1588230740 at 1733159749914Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1733159749929 (+15 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733159749929Flushing 1588230740/info: creating writer at 1733159749932 (+3 ms)Flushing 1588230740/info: appending metadata at 1733159750100 (+168 ms)Flushing 1588230740/info: closing flushed file at 1733159750106 (+6 ms)Flushing 1588230740/ns: creating writer at 1733159750308 (+202 ms)Flushing 1588230740/ns: appending metadata at 1733159750363 (+55 ms)Flushing 1588230740/ns: closing flushed file at 1733159750363Flushing 1588230740/table: creating writer at 1733159750530 (+167 ms)Flushing 1588230740/table: appending metadata at 1733159750566 (+36 ms)Flushing 1588230740/table: closing flushed file at 1733159750566Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a1be0b: reopening flushed file at 1733159750655 (+89 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79883cf7: reopening flushed file at 1733159750688 (+33 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42e9888: reopening flushed file at 1733159750703 (+15 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 822ms, sequenceid=60, compaction requested=false; wal=null at 1733159750735 (+32 ms)Cleaning up temporary data from old regions at 1733159750742 (+7 ms)Running coprocessor post-open hooks at 1733159750767 (+25 ms)Region opened successfully at 1733159750767 2024-12-02T17:15:50,771 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1733159749641 2024-12-02T17:15:50,776 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T17:15:50,776 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T17:15:50,777 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:50,779 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:50,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:50,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:50,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:50,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:50,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:50,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:50,781 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:50,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-02T17:15:50,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 in 1.3170 sec 2024-12-02T17:15:50,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-12-02T17:15:50,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.5030 sec 2024-12-02T17:15:50,793 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 3b3375d1a925,39557,1733159739502 had 9 regions 2024-12-02T17:15:50,795 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true, isMeta: false 2024-12-02T17:15:50,798 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3b3375d1a925,39557,1733159739502 WAL count=1, meta=false 2024-12-02T17:15:50,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597}] 2024-12-02T17:15:50,803 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:50,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:50,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-12-02T17:15:50,990 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597, size=2.2 K (2237bytes) 2024-12-02T17:15:50,990 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597 2024-12-02T17:15:50,991 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597 after 1ms 2024-12-02T17:15:50,996 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597 took 7ms 2024-12-02T17:15:51,010 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for af5ee9a539ef801e1148eebca0987b9c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T17:15:51,012 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 31473ece20fde70fc9741ad3ff42aadc: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T17:15:51,016 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d53e8d53f3fcf0c3420445811190e425: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T17:15:51,024 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ba8c728daa66bac733cfa12bcd38a313: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T17:15:51,027 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d612ab016a94fed4cbfb22a0c74d8f6a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T17:15:51,031 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 76faba7cbca7c506fec4bb6f534477a2: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T17:15:51,033 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a37d5d00fe28247586762a18d6e0943f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T17:15:51,037 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8981ccd8f09f082161872d7b109756f0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T17:15:51,037 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597 so closing down 2024-12-02T17:15:51,037 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T17:15:51,038 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T17:15:51,038 INFO [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 31 ms; skipped=8; WAL=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-12-02T17:15:51,038 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597, journal: Splitting hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597, size=2.2 K (2237bytes) at 1733159750990Finishing writing output for hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597 so closing down at 1733159751037 (+47 ms)3 split writer threads finished at 1733159751038 (+1 ms)Processed 8 edits across 0 Regions in 31 ms; skipped=8; WAL=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1733159751038 2024-12-02T17:15:51,038 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597 2024-12-02T17:15:51,038 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-02T17:15:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37351 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-02T17:15:51,051 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting/3b3375d1a925%2C39557%2C1733159739502.1733159742597 to hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs 2024-12-02T17:15:51,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-02T17:15:51,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 in 246 msec 2024-12-02T17:15:51,063 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-12-02T17:15:51,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 in 265 msec 2024-12-02T17:15:51,072 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/WALs/3b3375d1a925,39557,1733159739502-splitting dir is empty, no logs to split. 2024-12-02T17:15:51,072 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3b3375d1a925,39557,1733159739502 WAL count=0, meta=false 2024-12-02T17:15:51,072 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3b3375d1a925,39557,1733159739502 WAL splitting is done? wals=0, meta=false 2024-12-02T17:15:51,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN}] 2024-12-02T17:15:51,081 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:51,082 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T17:15:51,083 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:51,084 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T17:15:51,086 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:51,089 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T17:15:51,090 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:51,091 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T17:15:51,093 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:51,094 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T17:15:51,095 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:51,097 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T17:15:51,103 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:51,105 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T17:15:51,106 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:51,108 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T17:15:51,237 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(204): Hosts are {3b3375d1a925=0} racks are {/default-rack=0} 2024-12-02T17:15:51,237 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T17:15:51,237 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T17:15:51,237 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T17:15:51,237 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T17:15:51,237 INFO [3b3375d1a925:37351 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T17:15:51,237 INFO [3b3375d1a925:37351 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T17:15:51,237 DEBUG [3b3375d1a925:37351 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T17:15:51,240 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,242 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,242 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,243 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3b3375d1a925:39557 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3b3375d1a925/172.17.0.3:39557 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:51,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,39557,1733159739502, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,39557,1733159739502, seqNum=-1, error=java.net.ConnectException: Call to address=3b3375d1a925:39557 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3b3375d1a925/172.17.0.3:39557 2024-12-02T17:15:51,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,39557,1733159739502, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-02T17:15:51,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,39557,1733159739502, seqNum=-1 from cache 2024-12-02T17:15:51,248 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:51,248 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,33117,1733159739837, seqNum=-1] 2024-12-02T17:15:51,248 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:51,248 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, regionLocation=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, regionLocation=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,39557,1733159739502, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,33117,1733159739837, seqNum=-1, error=java.net.ConnectException: Call to address=3b3375d1a925:39557 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3b3375d1a925/172.17.0.3:39557 2024-12-02T17:15:51,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39429, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:51,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3b3375d1a925:39557 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3b3375d1a925/172.17.0.3:39557 2024-12-02T17:15:51,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN because future has completed 2024-12-02T17:15:51,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:51,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN because future has completed 2024-12-02T17:15:51,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN because future has completed 2024-12-02T17:15:51,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:51,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN because future has completed 2024-12-02T17:15:51,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN because future has completed 2024-12-02T17:15:51,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN because future has completed 2024-12-02T17:15:51,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:51,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:51,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:51,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN because future has completed 2024-12-02T17:15:51,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,34079,1733159740078}] 2024-12-02T17:15:51,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN because future has completed 2024-12-02T17:15:51,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:51,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d53e8d53f3fcf0c3420445811190e425, server=3b3375d1a925,33117,1733159739837}] 2024-12-02T17:15:51,437 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:51,437 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => a37d5d00fe28247586762a18d6e0943f, NAME => 'test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-02T17:15:51,438 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,438 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:51,438 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,438 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,454 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:51,454 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 76faba7cbca7c506fec4bb6f534477a2, NAME => 'test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-02T17:15:51,454 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,454 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:51,455 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,455 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,477 INFO [StoreOpener-76faba7cbca7c506fec4bb6f534477a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,487 INFO [StoreOpener-a37d5d00fe28247586762a18d6e0943f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,488 INFO [StoreOpener-76faba7cbca7c506fec4bb6f534477a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76faba7cbca7c506fec4bb6f534477a2 columnFamilyName family 2024-12-02T17:15:51,488 DEBUG [StoreOpener-76faba7cbca7c506fec4bb6f534477a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:51,489 INFO [StoreOpener-a37d5d00fe28247586762a18d6e0943f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a37d5d00fe28247586762a18d6e0943f columnFamilyName family 2024-12-02T17:15:51,489 INFO [StoreOpener-76faba7cbca7c506fec4bb6f534477a2-1 {}] regionserver.HStore(327): Store=76faba7cbca7c506fec4bb6f534477a2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:51,490 DEBUG [StoreOpener-a37d5d00fe28247586762a18d6e0943f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:51,490 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,491 INFO [StoreOpener-a37d5d00fe28247586762a18d6e0943f-1 {}] regionserver.HStore(327): Store=a37d5d00fe28247586762a18d6e0943f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:51,491 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,491 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,492 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,494 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,495 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,495 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,495 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,498 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,499 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,499 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened a37d5d00fe28247586762a18d6e0943f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71088808, jitterRate=0.059305787086486816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:51,500 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,500 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:51,500 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for a37d5d00fe28247586762a18d6e0943f: Running coprocessor pre-open hook at 1733159751438Writing region info on filesystem at 1733159751438Initializing all the Stores at 1733159751458 (+20 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159751458Cleaning up temporary data from old regions at 1733159751495 (+37 ms)Running coprocessor post-open hooks at 1733159751500 (+5 ms)Region opened successfully at 1733159751500 2024-12-02T17:15:51,502 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f., pid=75, masterSystemTime=1733159751427 2024-12-02T17:15:51,505 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,506 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 76faba7cbca7c506fec4bb6f534477a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67337439, jitterRate=0.003406032919883728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:51,506 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:51,506 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 76faba7cbca7c506fec4bb6f534477a2: Running coprocessor pre-open hook at 1733159751455Writing region info on filesystem at 1733159751455Initializing all the Stores at 1733159751458 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159751458Cleaning up temporary data from old regions at 1733159751500 (+42 ms)Running coprocessor post-open hooks at 1733159751506 (+6 ms)Region opened successfully at 1733159751506 2024-12-02T17:15:51,507 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2., pid=76, masterSystemTime=1733159751446 2024-12-02T17:15:51,510 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:51,511 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:51,511 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:51,511 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 31473ece20fde70fc9741ad3ff42aadc, NAME => 'test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-02T17:15:51,511 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,511 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:51,512 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,512 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,513 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:51,513 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:51,513 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:51,513 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => ba8c728daa66bac733cfa12bcd38a313, NAME => 'test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-02T17:15:51,513 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,513 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:51,513 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,514 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,524 INFO [StoreOpener-31473ece20fde70fc9741ad3ff42aadc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,526 INFO [StoreOpener-31473ece20fde70fc9741ad3ff42aadc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31473ece20fde70fc9741ad3ff42aadc columnFamilyName family 2024-12-02T17:15:51,526 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-02T17:15:51,526 DEBUG [StoreOpener-31473ece20fde70fc9741ad3ff42aadc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:51,526 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,527 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,533 INFO [StoreOpener-31473ece20fde70fc9741ad3ff42aadc-1 {}] regionserver.HStore(327): Store=31473ece20fde70fc9741ad3ff42aadc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:51,534 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,535 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,536 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,537 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,537 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,540 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,541 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 31473ece20fde70fc9741ad3ff42aadc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63303308, jitterRate=-0.05670720338821411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:51,541 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:51,542 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 31473ece20fde70fc9741ad3ff42aadc: Running coprocessor pre-open hook at 1733159751512Writing region info on filesystem at 1733159751512Initializing all the Stores at 1733159751514 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159751514Cleaning up temporary data from old regions at 1733159751537 (+23 ms)Running coprocessor post-open hooks at 1733159751541 (+4 ms)Region opened successfully at 1733159751541 2024-12-02T17:15:51,547 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc., pid=77, masterSystemTime=1733159751427 2024-12-02T17:15:51,549 INFO [StoreOpener-ba8c728daa66bac733cfa12bcd38a313-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,560 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:51,561 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:51,561 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:51,561 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => d612ab016a94fed4cbfb22a0c74d8f6a, NAME => 'test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-02T17:15:51,561 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,561 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:51,562 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,562 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,564 INFO [StoreOpener-ba8c728daa66bac733cfa12bcd38a313-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba8c728daa66bac733cfa12bcd38a313 columnFamilyName family 2024-12-02T17:15:51,564 DEBUG [StoreOpener-ba8c728daa66bac733cfa12bcd38a313-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:51,565 INFO [StoreOpener-ba8c728daa66bac733cfa12bcd38a313-1 {}] regionserver.HStore(327): Store=ba8c728daa66bac733cfa12bcd38a313/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:51,566 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,566 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,565 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-02T17:15:51,567 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:51,567 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d53e8d53f3fcf0c3420445811190e425, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:51,568 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d53e8d53f3fcf0c3420445811190e425, server=3b3375d1a925,33117,1733159739837 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,568 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,569 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,569 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,570 INFO [StoreOpener-d612ab016a94fed4cbfb22a0c74d8f6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,573 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,574 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened ba8c728daa66bac733cfa12bcd38a313; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60900795, jitterRate=-0.09250743687152863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:51,574 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:51,574 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for ba8c728daa66bac733cfa12bcd38a313: Running coprocessor pre-open hook at 1733159751514Writing region info on filesystem at 1733159751514Initializing all the Stores at 1733159751516 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159751516Cleaning up temporary data from old regions at 1733159751569 (+53 ms)Running coprocessor post-open hooks at 1733159751574 (+5 ms)Region opened successfully at 1733159751574 2024-12-02T17:15:51,576 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313., pid=73, masterSystemTime=1733159751446 2024-12-02T17:15:51,582 INFO [StoreOpener-d612ab016a94fed4cbfb22a0c74d8f6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d612ab016a94fed4cbfb22a0c74d8f6a columnFamilyName family 2024-12-02T17:15:51,582 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:51,582 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:51,582 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:51,582 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => af5ee9a539ef801e1148eebca0987b9c, NAME => 'test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-02T17:15:51,583 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,583 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:51,583 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,583 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,591 INFO [StoreOpener-af5ee9a539ef801e1148eebca0987b9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,593 DEBUG [StoreOpener-d612ab016a94fed4cbfb22a0c74d8f6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:51,594 INFO [StoreOpener-d612ab016a94fed4cbfb22a0c74d8f6a-1 {}] regionserver.HStore(327): Store=d612ab016a94fed4cbfb22a0c74d8f6a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:51,594 INFO [StoreOpener-af5ee9a539ef801e1148eebca0987b9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af5ee9a539ef801e1148eebca0987b9c columnFamilyName family 2024-12-02T17:15:51,594 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,594 DEBUG [StoreOpener-af5ee9a539ef801e1148eebca0987b9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:51,596 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,597 INFO [StoreOpener-af5ee9a539ef801e1148eebca0987b9c-1 {}] regionserver.HStore(327): Store=af5ee9a539ef801e1148eebca0987b9c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:51,597 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,598 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,601 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,601 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,602 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,602 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,602 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,602 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,605 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,606 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened af5ee9a539ef801e1148eebca0987b9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62894423, jitterRate=-0.06280006468296051}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:51,606 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:51,606 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for af5ee9a539ef801e1148eebca0987b9c: Running coprocessor pre-open hook at 1733159751583Writing region info on filesystem at 1733159751583Initializing all the Stores at 1733159751584 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159751584Cleaning up temporary data from old regions at 1733159751602 (+18 ms)Running coprocessor post-open hooks at 1733159751606 (+4 ms)Region opened successfully at 1733159751606 2024-12-02T17:15:51,608 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,608 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c., pid=78, masterSystemTime=1733159751446 2024-12-02T17:15:51,612 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened d612ab016a94fed4cbfb22a0c74d8f6a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59975515, jitterRate=-0.10629518330097198}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:51,612 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:51,612 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for d612ab016a94fed4cbfb22a0c74d8f6a: Running coprocessor pre-open hook at 1733159751562Writing region info on filesystem at 1733159751562Initializing all the Stores at 1733159751565 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159751566 (+1 ms)Cleaning up temporary data from old regions at 1733159751602 (+36 ms)Running coprocessor post-open hooks at 1733159751612 (+10 ms)Region opened successfully at 1733159751612 2024-12-02T17:15:51,614 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a., pid=72, masterSystemTime=1733159751427 2024-12-02T17:15:51,637 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:51,637 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:51,637 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:51,637 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => d53e8d53f3fcf0c3420445811190e425, NAME => 'test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-02T17:15:51,638 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,638 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:51,638 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,638 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,639 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:51,639 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:51,639 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:51,640 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:51,641 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:51,641 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:51,641 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:51,641 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 8981ccd8f09f082161872d7b109756f0, NAME => 'test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-02T17:15:51,642 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,642 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:51,642 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,642 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,647 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:51,650 INFO [StoreOpener-8981ccd8f09f082161872d7b109756f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,652 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:51,652 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:51,653 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:51,653 INFO [StoreOpener-8981ccd8f09f082161872d7b109756f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8981ccd8f09f082161872d7b109756f0 columnFamilyName family 2024-12-02T17:15:51,653 DEBUG [StoreOpener-8981ccd8f09f082161872d7b109756f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:51,654 INFO [StoreOpener-8981ccd8f09f082161872d7b109756f0-1 {}] regionserver.HStore(327): Store=8981ccd8f09f082161872d7b109756f0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:51,654 INFO [StoreOpener-d53e8d53f3fcf0c3420445811190e425-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,654 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,655 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,656 INFO [StoreOpener-d53e8d53f3fcf0c3420445811190e425-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d53e8d53f3fcf0c3420445811190e425 columnFamilyName family 2024-12-02T17:15:51,656 DEBUG [StoreOpener-d53e8d53f3fcf0c3420445811190e425-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:51,657 INFO [StoreOpener-d53e8d53f3fcf0c3420445811190e425-1 {}] regionserver.HStore(327): Store=d53e8d53f3fcf0c3420445811190e425/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:51,657 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,658 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,659 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,660 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,660 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 7 msec 2024-12-02T17:15:51,661 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,662 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,662 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,665 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,668 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,669 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 8981ccd8f09f082161872d7b109756f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73340407, jitterRate=0.092857226729393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:51,669 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:51,669 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 8981ccd8f09f082161872d7b109756f0: Running coprocessor pre-open hook at 1733159751642Writing region info on filesystem at 1733159751642Initializing all the Stores at 1733159751647 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159751647Cleaning up temporary data from old regions at 1733159751660 (+13 ms)Running coprocessor post-open hooks at 1733159751669 (+9 ms)Region opened successfully at 1733159751669 2024-12-02T17:15:51,669 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened d53e8d53f3fcf0c3420445811190e425; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64690106, jitterRate=-0.03604230284690857}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T17:15:51,670 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:51,670 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for d53e8d53f3fcf0c3420445811190e425: Running coprocessor pre-open hook at 1733159751638Writing region info on filesystem at 1733159751638Initializing all the Stores at 1733159751641 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159751641Cleaning up temporary data from old regions at 1733159751662 (+21 ms)Running coprocessor post-open hooks at 1733159751670 (+8 ms)Region opened successfully at 1733159751670 2024-12-02T17:15:51,672 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425., pid=79, masterSystemTime=1733159751446 2024-12-02T17:15:51,673 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,673 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0., pid=74, masterSystemTime=1733159751427 2024-12-02T17:15:51,674 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,675 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "3b3375d1a925" port: 33117 start_code: 1733159739837 } transition { transition_code: OPENED region_info { region_id: 1733159743838 table_name { namespace: "default" qualifier: "test" } start_key: "xxx" end_key: "yyy" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1733159740270 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-02T17:15:51,675 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "3b3375d1a925" port: 34079 start_code: 1733159740078 } transition { transition_code: OPENED region_info { region_id: 1733159743838 table_name { namespace: "default" qualifier: "test" } start_key: "eee" end_key: "fff" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 74 initiating_master_active_time: 1733159740270 }; retry (#0) after 1000ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-02T17:15:51,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:51,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:51,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:51,690 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:51,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:51,693 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,703 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:51,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d53e8d53f3fcf0c3420445811190e425, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:51,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,713 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:51,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d53e8d53f3fcf0c3420445811190e425, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,715 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 54 msec 2024-12-02T17:15:51,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:51,723 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:51,724 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:51,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:51,728 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:51,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:51,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:51,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:51,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:51,739 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:51,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:51,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:51,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:51,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:51,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:51,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:51,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN to state=OFFLINE, location=null, table=test, region=31473ece20fde70fc9741ad3ff42aadc to restore RIT 2024-12-02T17:15:51,769 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159781769 2024-12-02T17:15:51,770 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:51,771 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,784 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:51,786 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:51,786 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:51,786 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,33117,1733159739837 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,857 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:51,857 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159811857 2024-12-02T17:15:51,857 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159871857 2024-12-02T17:15:51,857 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 87 msec 2024-12-02T17:15:51,857 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:51,857 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:51,857 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:51,857 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:51,858 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:51,858 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:51,858 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:51,858 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:51,862 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,876 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:51,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:51,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:51,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:51,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:51,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:51,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,885 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,885 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,885 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,885 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:51,885 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,885 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,886 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,886 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,886 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure af5ee9a539ef801e1148eebca0987b9c, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,886 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 23 msec 2024-12-02T17:15:51,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:51,888 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:51,888 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:51,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:51,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:51,891 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:51,891 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:51,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:51,893 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:51,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:51,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:51,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:51,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:51,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:51,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:51,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:51,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN to state=OFFLINE, location=null, table=test, region=31473ece20fde70fc9741ad3ff42aadc to restore RIT 2024-12-02T17:15:51,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159781900 2024-12-02T17:15:51,901 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:51,905 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,907 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:51,908 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:51,908 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:51,908 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,34079,1733159740078 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,920 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:51,921 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159811921 2024-12-02T17:15:51,921 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159871921 2024-12-02T17:15:51,921 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 20 msec 2024-12-02T17:15:51,921 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:51,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:51,921 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:51,921 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:51,926 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:51,926 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:51,926 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:51,926 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:51,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:51,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:51,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:51,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:51,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:51,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:51,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:51,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:51,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:51,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:51,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:51,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:51,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:51,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:51,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,952 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,953 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,953 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,953 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,953 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,953 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,953 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,953 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,953 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,953 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,954 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,954 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31473ece20fde70fc9741ad3ff42aadc, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,954 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 26 msec 2024-12-02T17:15:51,957 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:51,958 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:51,958 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:51,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:51,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:51,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:51,960 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:51,960 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:51,960 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:51,961 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:51,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:51,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:51,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:51,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:51,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:51,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:51,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN to state=OFFLINE, location=null, table=test, region=31473ece20fde70fc9741ad3ff42aadc to restore RIT 2024-12-02T17:15:51,968 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159781968 2024-12-02T17:15:51,968 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:51,969 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,970 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:51,975 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:51,975 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:51,977 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:51,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:51,986 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:51,986 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159811986 2024-12-02T17:15:51,986 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159871986 2024-12-02T17:15:51,986 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 17 msec 2024-12-02T17:15:51,986 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:51,986 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:51,986 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:51,987 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:51,987 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:51,987 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:51,987 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:51,987 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:51,988 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:51,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:51,993 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:51,993 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:51,993 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:51,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:51,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:51,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:51,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:51,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76faba7cbca7c506fec4bb6f534477a2, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,001 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-12-02T17:15:52,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,005 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,005 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,007 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,007 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,008 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:52,009 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:52,009 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:52,009 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN to state=OFFLINE, location=null, table=test, region=31473ece20fde70fc9741ad3ff42aadc to restore RIT 2024-12-02T17:15:52,011 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782011 2024-12-02T17:15:52,011 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,011 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,013 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,022 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,022 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,022 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,037 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,037 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812037 2024-12-02T17:15:52,037 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872037 2024-12-02T17:15:52,037 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-02T17:15:52,037 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,037 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,037 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,038 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,038 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,038 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,038 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,039 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,048 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 9 msec 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,063 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-12-02T17:15:52,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,064 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,064 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,067 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,067 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,067 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:52,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:52,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:52,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN to state=OFFLINE, location=null, table=test, region=31473ece20fde70fc9741ad3ff42aadc to restore RIT 2024-12-02T17:15:52,071 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782071 2024-12-02T17:15:52,071 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,072 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,074 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,084 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812084 2024-12-02T17:15:52,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872084 2024-12-02T17:15:52,085 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-12-02T17:15:52,085 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,085 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,335 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,335 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,335 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,336 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,336 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,337 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,337 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,337 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,338 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,341 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,342 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,342 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,342 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,342 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,343 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,346 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,346 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,347 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,347 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,347 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,347 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,347 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,347 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,347 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,347 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,347 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,347 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,347 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,347 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,347 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,347 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,347 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,347 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8981ccd8f09f082161872d7b109756f0, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-02T17:15:52,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,348 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,349 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:52,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:52,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:52,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN to state=OFFLINE, location=null, table=test, region=31473ece20fde70fc9741ad3ff42aadc to restore RIT 2024-12-02T17:15:52,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,353 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,353 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782353 2024-12-02T17:15:52,353 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,355 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,356 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,361 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,361 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,361 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,366 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,366 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812366 2024-12-02T17:15:52,366 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872366 2024-12-02T17:15:52,367 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-12-02T17:15:52,367 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,367 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,367 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,367 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,367 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,367 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,367 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,368 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,368 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,377 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,377 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,377 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,377 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,377 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,377 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,377 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,377 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,377 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,377 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,377 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,377 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,377 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,377 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,377 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,377 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,377 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba8c728daa66bac733cfa12bcd38a313, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,377 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-02T17:15:52,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,385 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,385 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,388 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,388 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,388 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,390 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,390 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,390 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,390 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,390 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:52,390 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:52,390 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:52,390 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN to state=OFFLINE, location=null, table=test, region=31473ece20fde70fc9741ad3ff42aadc to restore RIT 2024-12-02T17:15:52,401 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782401 2024-12-02T17:15:52,401 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,401 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,403 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,406 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,406 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,406 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,417 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,417 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812417 2024-12-02T17:15:52,417 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872417 2024-12-02T17:15:52,417 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-12-02T17:15:52,417 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,417 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,417 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,418 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,422 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,422 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,423 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,425 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,429 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,429 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,429 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,429 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,429 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,429 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,431 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,435 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,435 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,435 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,435 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,435 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,437 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,437 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,437 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,437 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,438 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,438 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,438 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,438 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,438 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,438 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,438 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,438 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,438 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,438 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d612ab016a94fed4cbfb22a0c74d8f6a, server=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,438 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-12-02T17:15:52,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,439 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,439 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,441 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,442 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,446 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,449 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,453 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,455 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,455 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,457 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,457 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,457 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:52,457 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:52,457 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:52,457 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN to state=OFFLINE, location=null, table=test, region=31473ece20fde70fc9741ad3ff42aadc to restore RIT 2024-12-02T17:15:52,459 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782459 2024-12-02T17:15:52,459 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,460 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,462 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,474 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,474 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,475 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,475 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,475 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812475 2024-12-02T17:15:52,476 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872475 2024-12-02T17:15:52,476 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-12-02T17:15:52,476 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,476 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,476 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,476 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,476 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,476 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,476 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,477 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,478 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,480 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,485 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,485 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN 2024-12-02T17:15:52,485 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-02T17:15:52,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,491 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,491 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,495 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,497 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,497 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,497 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,497 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,497 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:52,497 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:52,497 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:52,497 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=31473ece20fde70fc9741ad3ff42aadc, ASSIGN to state=OFFLINE, location=null, table=test, region=31473ece20fde70fc9741ad3ff42aadc to restore RIT 2024-12-02T17:15:52,499 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782499 2024-12-02T17:15:52,499 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,500 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,501 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,504 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,504 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,504 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,514 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,514 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812514 2024-12-02T17:15:52,514 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872514 2024-12-02T17:15:52,514 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-12-02T17:15:52,514 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,514 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,514 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,515 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,515 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,515 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,515 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,515 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,516 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,518 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,522 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,522 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,522 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,522 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,522 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,522 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,522 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,522 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,522 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,522 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,522 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,522 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,522 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,522 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,522 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN 2024-12-02T17:15:52,522 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-02T17:15:52,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,529 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,530 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,535 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,535 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,535 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,535 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,535 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:52,535 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:52,535 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76faba7cbca7c506fec4bb6f534477a2, ASSIGN to state=OFFLINE, location=null, table=test, region=76faba7cbca7c506fec4bb6f534477a2 to restore RIT 2024-12-02T17:15:52,540 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782539 2024-12-02T17:15:52,540 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,550 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,550 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812550 2024-12-02T17:15:52,550 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872550 2024-12-02T17:15:52,550 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-02T17:15:52,542 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,552 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,553 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,553 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,554 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,678 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.TransitRegionStateProcedure(535): There is no outstanding remote region procedure for state=OPENING, location=3b3375d1a925,34079,1733159740078, table=test, region=8981ccd8f09f082161872d7b109756f0, serverName=3b3375d1a925,34079,1733159740078, code=OPENED, seqId=2, proc=pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN, should be a retry, ignore 2024-12-02T17:15:52,679 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2353): TRANSITION REPORTED server { host_name: "3b3375d1a925" port: 34079 start_code: 1733159740078 } transition { transition_code: OPENED region_info { region_id: 1733159743838 table_name { namespace: "default" qualifier: "test" } start_key: "eee" end_key: "fff" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 74 initiating_master_active_time: 1733159740270 } 2024-12-02T17:15:52,679 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:52,680 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:52,685 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] assignment.TransitRegionStateProcedure(535): There is no outstanding remote region procedure for state=OPENING, location=3b3375d1a925,33117,1733159739837, table=test, region=d53e8d53f3fcf0c3420445811190e425, serverName=3b3375d1a925,33117,1733159739837, code=OPENED, seqId=2, proc=pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN, should be a retry, ignore 2024-12-02T17:15:52,686 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2353): TRANSITION REPORTED server { host_name: "3b3375d1a925" port: 33117 start_code: 1733159739837 } transition { transition_code: OPENED region_info { region_id: 1733159743838 table_name { namespace: "default" qualifier: "test" } start_key: "xxx" end_key: "yyy" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1733159740270 } 2024-12-02T17:15:52,686 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:52,686 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:52,801 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,801 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,801 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,801 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,801 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,801 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,801 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,801 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,802 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,805 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,806 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,806 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,806 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,806 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,806 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,807 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,807 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,807 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN 2024-12-02T17:15:52,807 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T17:15:52,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,807 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,807 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:52,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d53e8d53f3fcf0c3420445811190e425, ASSIGN to state=OFFLINE, location=null, table=test, region=d53e8d53f3fcf0c3420445811190e425 to restore RIT 2024-12-02T17:15:52,810 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782810 2024-12-02T17:15:52,811 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,811 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,812 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,813 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,813 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,813 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,818 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,818 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812818 2024-12-02T17:15:52,818 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872818 2024-12-02T17:15:52,818 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-02T17:15:52,818 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,818 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,818 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,818 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,819 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,819 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,819 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,819 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,819 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN 2024-12-02T17:15:52,825 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-02T17:15:52,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,825 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,825 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,828 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,828 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,828 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,828 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,828 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8981ccd8f09f082161872d7b109756f0, ASSIGN to state=OFFLINE, location=null, table=test, region=8981ccd8f09f082161872d7b109756f0 to restore RIT 2024-12-02T17:15:52,829 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782829 2024-12-02T17:15:52,829 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,829 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,831 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,832 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,832 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,832 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,836 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,836 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812836 2024-12-02T17:15:52,836 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872836 2024-12-02T17:15:52,836 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-02T17:15:52,836 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,837 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,837 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,837 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,837 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,837 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,837 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,837 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,838 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,839 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,842 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,842 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN 2024-12-02T17:15:52,843 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T17:15:52,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,844 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,844 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,846 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,846 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,846 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,846 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,847 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,847 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,847 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a, ASSIGN to state=OFFLINE, location=null, table=test, region=d612ab016a94fed4cbfb22a0c74d8f6a to restore RIT 2024-12-02T17:15:52,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782847 2024-12-02T17:15:52,847 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,847 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,848 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,849 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,849 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,849 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,855 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812855 2024-12-02T17:15:52,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872855 2024-12-02T17:15:52,855 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-02T17:15:52,855 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:52,855 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,855 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:52,856 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:52,856 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:52,856 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:52,856 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:52,856 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:52,857 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:52,858 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:52,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:52,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:52,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:52,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:52,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:52,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:52,862 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,862 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:52,862 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,862 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,862 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:52,862 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN 2024-12-02T17:15:52,862 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T17:15:52,862 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,863 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:52,863 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:52,863 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:52,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:52,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:52,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:52,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:52,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba8c728daa66bac733cfa12bcd38a313, ASSIGN to state=OFFLINE, location=null, table=test, region=ba8c728daa66bac733cfa12bcd38a313 to restore RIT 2024-12-02T17:15:52,866 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159782866 2024-12-02T17:15:52,866 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:52,866 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:52,868 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:52,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:52,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:52,874 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:52,874 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159812874 2024-12-02T17:15:52,874 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159872874 2024-12-02T17:15:52,874 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-02T17:15:52,875 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:52,875 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:52,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:52,875 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,125 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:53,125 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,125 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,125 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,125 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:53,125 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:53,125 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:53,126 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:53,127 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T17:15:53,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:53,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:53,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:53,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:53,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:53,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:53,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:53,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:53,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:53,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:53,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN 2024-12-02T17:15:53,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T17:15:53,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:53,133 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:53,133 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:53,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:53,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af5ee9a539ef801e1148eebca0987b9c, ASSIGN to state=OFFLINE, location=null, table=test, region=af5ee9a539ef801e1148eebca0987b9c to restore RIT 2024-12-02T17:15:53,141 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159783141 2024-12-02T17:15:53,141 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:53,142 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,144 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:53,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:53,145 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:53,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,146 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,151 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:53,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159813151 2024-12-02T17:15:53,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159873151 2024-12-02T17:15:53,151 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-02T17:15:53,151 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:53,151 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,151 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,151 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,151 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:53,151 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:53,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:53,152 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:53,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:53,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:53,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:53,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:53,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:53,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:53,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN 2024-12-02T17:15:53,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T17:15:53,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:53,159 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:53,159 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:53,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,162 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a37d5d00fe28247586762a18d6e0943f, ASSIGN to state=OFFLINE, location=null, table=test, region=a37d5d00fe28247586762a18d6e0943f to restore RIT 2024-12-02T17:15:53,162 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159783162 2024-12-02T17:15:53,162 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:53,162 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,164 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:53,165 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:53,165 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,165 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,168 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:53,168 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159813168 2024-12-02T17:15:53,168 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159873168 2024-12-02T17:15:53,168 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-02T17:15:53,168 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:53,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,168 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,168 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,169 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:53,169 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:53,169 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:53,169 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:53,169 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:53,171 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:53,171 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,171 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,171 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:53,171 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:53,171 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,171 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:53,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,172 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,172 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,173 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,173 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,173 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,173 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,173 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-02T17:15:53,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:53,173 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:53,173 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:53,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,175 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,175 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,181 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159783181 2024-12-02T17:15:53,181 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:53,181 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,183 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,188 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,188 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,188 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,189 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:53,189 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159813189 2024-12-02T17:15:53,189 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159873189 2024-12-02T17:15:53,189 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-02T17:15:53,189 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:53,189 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,189 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,189 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,189 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:53,189 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:53,189 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:53,190 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:53,190 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:53,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:53,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:53,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:53,192 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:53,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.1733159742597, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-02T17:15:53,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:53,194 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:53,194 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:53,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159783200 2024-12-02T17:15:53,201 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:53,201 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,204 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:53,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,205 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:53,205 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:53,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,205 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,207 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:53,207 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159813207 2024-12-02T17:15:53,207 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159873207 2024-12-02T17:15:53,207 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-02T17:15:53,207 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:53,207 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,207 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,207 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,207 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:53,207 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:53,207 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:53,207 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:53,208 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:53,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:53,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:53,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T17:15:53,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:53,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:53,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:53,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 2024-12-02T17:15:53,210 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,33117,1733159739837 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,211 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-02T17:15:53,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:53,211 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:53,211 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:53,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,217 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159783217 2024-12-02T17:15:53,217 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:53,217 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,218 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:53,219 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:53,219 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:53,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,219 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,225 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:53,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159813225 2024-12-02T17:15:53,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159873225 2024-12-02T17:15:53,225 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-02T17:15:53,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:53,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,225 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,225 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:53,225 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:53,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:53,225 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:53,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:53,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:53,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,228 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,228 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T17:15:53,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:53,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:53,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:15:53,229 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,229 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,229 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,229 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,229 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-02T17:15:53,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:53,230 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:53,230 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:53,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,233 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159783233 2024-12-02T17:15:53,233 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:53,233 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,233 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:53,234 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:53,234 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,235 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,238 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:53,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159813238 2024-12-02T17:15:53,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159873238 2024-12-02T17:15:53,238 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-02T17:15:53,238 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:53,238 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,238 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,239 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,239 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:53,239 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:53,239 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:53,239 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:53,240 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:53,241 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:53,241 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,241 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T17:15:53,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:53,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:53,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,242 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-02T17:15:53,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:53,242 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:53,242 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:53,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,244 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,244 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,244 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,245 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159783245 2024-12-02T17:15:53,245 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:53,245 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,246 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:53,247 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:53,247 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,247 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,250 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:53,250 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159813250 2024-12-02T17:15:53,250 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159873250 2024-12-02T17:15:53,250 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-12-02T17:15:53,250 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:53,250 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,250 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,250 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,250 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:53,250 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:53,250 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:53,250 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:53,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:53,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:53,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T17:15:53,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:53,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:53,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,253 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,253 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3b3375d1a925%2C39557%2C1733159739502.meta.1733159743065.meta, worker=3b3375d1a925,34079,1733159740078 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T17:15:53,253 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 1 msec 2024-12-02T17:15:53,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:53,253 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:53,253 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:53,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,256 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159783256 2024-12-02T17:15:53,256 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:53,256 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,257 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:53,258 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T17:15:53,259 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,259 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,263 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:53,263 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159813263 2024-12-02T17:15:53,263 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159873263 2024-12-02T17:15:53,263 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-02T17:15:53,263 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T17:15:53,263 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,263 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,263 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,263 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T17:15:53,263 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T17:15:53,263 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T17:15:53,263 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:15:53,264 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T17:15:53,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T17:15:53,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:15:53,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T17:15:53,266 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true 2024-12-02T17:15:53,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T17:15:53,266 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 1 msec 2024-12-02T17:15:53,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=61 2024-12-02T17:15:53,266 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3b3375d1a925,33117,1733159739837, table=hbase:meta, region=1588230740 2024-12-02T17:15:53,266 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,33117,1733159739837, state=OPEN 2024-12-02T17:15:53,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,268 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,268 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:15:53,268 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:15:53,269 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159783269 2024-12-02T17:15:53,269 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:15:53,269 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:53,270 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T17:15:53,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure a37d5d00fe28247586762a18d6e0943f, server=3b3375d1a925,34079,1733159740078:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3b3375d1a925,39557,1733159739502, splitWal=true, meta=true exec-time=4.5120 sec 2024-12-02T17:15:53,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16d4fe4d0593f960b6c0f8237bf0711b, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc881fc21c14f92b22d6d3f233ac48b1, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf8dc4ff69f7afd79a39737b7174af90, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee07072225557a5ffccbd5b4dc3f8250, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8981ccd8f09f082161872d7b109756f0, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=846f82385af02c9a3c05149f7ca40c3a, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76faba7cbca7c506fec4bb6f534477a2, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf990684b331d6bf1770501628bfa34f, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eba2814ba0cf502df62df55d77023b23, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77b4e8908ad2baccd7eb5d5fa846dff8, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=31473ece20fde70fc9741ad3ff42aadc, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80be845edde76a33d61215cef1be07d7, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba8c728daa66bac733cfa12bcd38a313, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8a17d43fe45538f8e98a951b88f67de, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd20102809dc64a92201c4ba1e20e38, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d612ab016a94fed4cbfb22a0c74d8f6a, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd047fbb6ae4c8ccd6da6584c52f6db, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af5ee9a539ef801e1148eebca0987b9c, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a37d5d00fe28247586762a18d6e0943f, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd24aed41e647f8e01f1c4bdbf2c148, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4eb95452ea2c6ad04c7114e1401424b9, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fd27a546540e9b8b8f7a078f3515500, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e940693c5f493c59a469d01db516d5, regionState=OPEN, lastHost=3b3375d1a925,34079,1733159740078, regionLocation=3b3375d1a925,34079,1733159740078, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d53e8d53f3fcf0c3420445811190e425, regionState=OPENING, lastHost=3b3375d1a925,39557,1733159739502, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3882d8b2ab81cfedea2b9eda817f13bf, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b72b8ea0bceb0151b39bea3aa7f2a9e, regionState=OPEN, lastHost=3b3375d1a925,33117,1733159739837, regionLocation=3b3375d1a925,33117,1733159739837, openSeqNum=2 2024-12-02T17:15:53,275 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T17:15:53,275 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159813275 2024-12-02T17:15:53,275 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159873275 2024-12-02T17:15:53,275 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-02T17:15:53,294 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=345 (was 378), OpenFileDescriptor=624 (was 647), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=976 (was 1043), ProcessCount=11 (was 11), AvailableMemoryMB=3233 (was 3575) 2024-12-02T17:15:53,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T17:15:53,295 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:15:53,296 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:15:53,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:53,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:53,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:53,298 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T17:15:53,298 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1872569587, stopped=false 2024-12-02T17:15:53,298 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3b3375d1a925,37351,1733159738385 2024-12-02T17:15:53,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T17:15:53,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:53,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T17:15:53,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:53,300 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,301 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:15:53,301 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:15:53,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:53,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T17:15:53,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:53,301 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:15:53,302 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:15:53,302 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:15:53,302 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3b3375d1a925,33117,1733159739837' ***** 2024-12-02T17:15:53,303 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T17:15:53,303 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3b3375d1a925,34079,1733159740078' ***** 2024-12-02T17:15:53,304 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T17:15:53,304 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T17:15:53,304 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T17:15:53,304 INFO [RS:1;3b3375d1a925:33117 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T17:15:53,304 INFO [RS:2;3b3375d1a925:34079 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T17:15:53,304 INFO [RS:1;3b3375d1a925:33117 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T17:15:53,304 INFO [RS:2;3b3375d1a925:34079 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T17:15:53,304 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T17:15:53,304 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for 0fd047fbb6ae4c8ccd6da6584c52f6db 2024-12-02T17:15:53,304 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:53,304 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for 1fd27a546540e9b8b8f7a078f3515500 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for af5ee9a539ef801e1148eebca0987b9c 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for 6dd20102809dc64a92201c4ba1e20e38 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for 76faba7cbca7c506fec4bb6f534477a2 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for ba8c728daa66bac733cfa12bcd38a313 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for 5b72b8ea0bceb0151b39bea3aa7f2a9e 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for d53e8d53f3fcf0c3420445811190e425 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for 80be845edde76a33d61215cef1be07d7 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for 3882d8b2ab81cfedea2b9eda817f13bf 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for 846f82385af02c9a3c05149f7ca40c3a 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(959): stopping server 3b3375d1a925,33117,1733159739837 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T17:15:53,305 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0fd047fbb6ae4c8ccd6da6584c52f6db, disabling compactions & flushes 2024-12-02T17:15:53,305 INFO [RS:1;3b3375d1a925:33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3b3375d1a925:33117. 2024-12-02T17:15:53,305 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:53,305 DEBUG [RS:1;3b3375d1a925:33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:15:53,305 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:53,305 DEBUG [RS:1;3b3375d1a925:33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:53,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. after waiting 0 ms 2024-12-02T17:15:53,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:53,306 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T17:15:53,306 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T17:15:53,306 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T17:15:53,306 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T17:15:53,306 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for d612ab016a94fed4cbfb22a0c74d8f6a 2024-12-02T17:15:53,306 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for 77b4e8908ad2baccd7eb5d5fa846dff8 2024-12-02T17:15:53,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fc881fc21c14f92b22d6d3f233ac48b1, disabling compactions & flushes 2024-12-02T17:15:53,306 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for 31473ece20fde70fc9741ad3ff42aadc 2024-12-02T17:15:53,306 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:53,306 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for 51e940693c5f493c59a469d01db516d5 2024-12-02T17:15:53,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:53,306 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for 16d4fe4d0593f960b6c0f8237bf0711b 2024-12-02T17:15:53,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. after waiting 0 ms 2024-12-02T17:15:53,306 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for 4eb95452ea2c6ad04c7114e1401424b9 2024-12-02T17:15:53,307 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for e8a17d43fe45538f8e98a951b88f67de 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for 8cd24aed41e647f8e01f1c4bdbf2c148 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for ee07072225557a5ffccbd5b4dc3f8250 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for cf990684b331d6bf1770501628bfa34f 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(3091): Received CLOSE for 8981ccd8f09f082161872d7b109756f0 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(959): stopping server 3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;3b3375d1a925:34079. 2024-12-02T17:15:53,307 DEBUG [RS:2;3b3375d1a925:34079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:15:53,307 DEBUG [RS:2;3b3375d1a925:34079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:53,307 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-12-02T17:15:53,308 DEBUG [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(1351): Waiting on 16d4fe4d0593f960b6c0f8237bf0711b, 31473ece20fde70fc9741ad3ff42aadc, 4eb95452ea2c6ad04c7114e1401424b9, 51e940693c5f493c59a469d01db516d5, 77b4e8908ad2baccd7eb5d5fa846dff8, 8981ccd8f09f082161872d7b109756f0, 8cd24aed41e647f8e01f1c4bdbf2c148, a37d5d00fe28247586762a18d6e0943f, cf990684b331d6bf1770501628bfa34f, d612ab016a94fed4cbfb22a0c74d8f6a, e8a17d43fe45538f8e98a951b88f67de, ee07072225557a5ffccbd5b4dc3f8250, fc881fc21c14f92b22d6d3f233ac48b1 2024-12-02T17:15:53,312 INFO [regionserver/3b3375d1a925:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T17:15:53,312 INFO [regionserver/3b3375d1a925:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T17:15:53,312 INFO [regionserver/3b3375d1a925:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T17:15:53,312 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-12-02T17:15:53,312 DEBUG [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(1351): Waiting on 0fd047fbb6ae4c8ccd6da6584c52f6db, 1588230740, 1fd27a546540e9b8b8f7a078f3515500, 3882d8b2ab81cfedea2b9eda817f13bf, 5b72b8ea0bceb0151b39bea3aa7f2a9e, 6dd20102809dc64a92201c4ba1e20e38, 76faba7cbca7c506fec4bb6f534477a2, 80be845edde76a33d61215cef1be07d7, 846f82385af02c9a3c05149f7ca40c3a, af5ee9a539ef801e1148eebca0987b9c, ba8c728daa66bac733cfa12bcd38a313, bf8dc4ff69f7afd79a39737b7174af90, d53e8d53f3fcf0c3420445811190e425, eba2814ba0cf502df62df55d77023b23 2024-12-02T17:15:53,313 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T17:15:53,313 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T17:15:53,313 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T17:15:53,313 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T17:15:53,313 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T17:15:53,313 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-12-02T17:15:53,321 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/0fd047fbb6ae4c8ccd6da6584c52f6db/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,321 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/fc881fc21c14f92b22d6d3f233ac48b1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,324 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:53,324 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:53,324 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fc881fc21c14f92b22d6d3f233ac48b1: Waiting for close lock at 1733159753306Running coprocessor pre-close hooks at 1733159753306Disabling compacts and flushes for region at 1733159753306Disabling writes for close at 1733159753307 (+1 ms)Writing region close event to WAL at 1733159753313 (+6 ms)Running coprocessor post-close hooks at 1733159753322 (+9 ms)Closed at 1733159753324 (+2 ms) 2024-12-02T17:15:53,324 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0fd047fbb6ae4c8ccd6da6584c52f6db: Waiting for close lock at 1733159753305Running coprocessor pre-close hooks at 1733159753305Disabling compacts and flushes for region at 1733159753305Disabling writes for close at 1733159753306 (+1 ms)Writing region close event to WAL at 1733159753308 (+2 ms)Running coprocessor post-close hooks at 1733159753322 (+14 ms)Closed at 1733159753324 (+2 ms) 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1733159743838.0fd047fbb6ae4c8ccd6da6584c52f6db. 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733159743838.fc881fc21c14f92b22d6d3f233ac48b1. 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1fd27a546540e9b8b8f7a078f3515500, disabling compactions & flushes 2024-12-02T17:15:53,325 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d612ab016a94fed4cbfb22a0c74d8f6a, disabling compactions & flushes 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. after waiting 0 ms 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:53,325 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. after waiting 0 ms 2024-12-02T17:15:53,325 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:53,326 INFO [regionserver/3b3375d1a925:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T17:15:53,339 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/1fd27a546540e9b8b8f7a078f3515500/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,340 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:53,340 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1fd27a546540e9b8b8f7a078f3515500: Waiting for close lock at 1733159753325Running coprocessor pre-close hooks at 1733159753325Disabling compacts and flushes for region at 1733159753325Disabling writes for close at 1733159753325Writing region close event to WAL at 1733159753325Running coprocessor post-close hooks at 1733159753340 (+15 ms)Closed at 1733159753340 2024-12-02T17:15:53,344 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1733159743838.1fd27a546540e9b8b8f7a078f3515500. 2024-12-02T17:15:53,344 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing af5ee9a539ef801e1148eebca0987b9c, disabling compactions & flushes 2024-12-02T17:15:53,344 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:53,344 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:53,344 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. after waiting 0 ms 2024-12-02T17:15:53,344 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:53,346 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d612ab016a94fed4cbfb22a0c74d8f6a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,347 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:53,347 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d612ab016a94fed4cbfb22a0c74d8f6a: Waiting for close lock at 1733159753325Running coprocessor pre-close hooks at 1733159753325Disabling compacts and flushes for region at 1733159753325Disabling writes for close at 1733159753325Writing region close event to WAL at 1733159753333 (+8 ms)Running coprocessor post-close hooks at 1733159753347 (+14 ms)Closed at 1733159753347 2024-12-02T17:15:53,347 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1733159743838.d612ab016a94fed4cbfb22a0c74d8f6a. 2024-12-02T17:15:53,347 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 77b4e8908ad2baccd7eb5d5fa846dff8, disabling compactions & flushes 2024-12-02T17:15:53,347 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:53,347 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:53,347 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. after waiting 0 ms 2024-12-02T17:15:53,347 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:53,348 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/info/40852d8bd0654c2cbd8ad43eacc7fa4e is 138, key is test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0./info:regioninfo/1733159751251/Put/seqid=0 2024-12-02T17:15:53,360 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/af5ee9a539ef801e1148eebca0987b9c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,361 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:53,361 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for af5ee9a539ef801e1148eebca0987b9c: Waiting for close lock at 1733159753344Running coprocessor pre-close hooks at 1733159753344Disabling compacts and flushes for region at 1733159753344Disabling writes for close at 1733159753344Writing region close event to WAL at 1733159753345 (+1 ms)Running coprocessor post-close hooks at 1733159753361 (+16 ms)Closed at 1733159753361 2024-12-02T17:15:53,361 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733159743838.af5ee9a539ef801e1148eebca0987b9c. 2024-12-02T17:15:53,361 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6dd20102809dc64a92201c4ba1e20e38, disabling compactions & flushes 2024-12-02T17:15:53,361 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:53,361 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:53,361 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. after waiting 0 ms 2024-12-02T17:15:53,361 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:53,366 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/77b4e8908ad2baccd7eb5d5fa846dff8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,366 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:53,366 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 77b4e8908ad2baccd7eb5d5fa846dff8: Waiting for close lock at 1733159753347Running coprocessor pre-close hooks at 1733159753347Disabling compacts and flushes for region at 1733159753347Disabling writes for close at 1733159753347Writing region close event to WAL at 1733159753353 (+6 ms)Running coprocessor post-close hooks at 1733159753366 (+13 ms)Closed at 1733159753366 2024-12-02T17:15:53,367 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733159743838.77b4e8908ad2baccd7eb5d5fa846dff8. 2024-12-02T17:15:53,367 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 31473ece20fde70fc9741ad3ff42aadc, disabling compactions & flushes 2024-12-02T17:15:53,367 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:53,367 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:53,367 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. after waiting 0 ms 2024-12-02T17:15:53,367 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:53,376 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/6dd20102809dc64a92201c4ba1e20e38/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,377 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:53,377 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6dd20102809dc64a92201c4ba1e20e38: Waiting for close lock at 1733159753361Running coprocessor pre-close hooks at 1733159753361Disabling compacts and flushes for region at 1733159753361Disabling writes for close at 1733159753361Writing region close event to WAL at 1733159753368 (+7 ms)Running coprocessor post-close hooks at 1733159753377 (+9 ms)Closed at 1733159753377 2024-12-02T17:15:53,377 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733159743838.6dd20102809dc64a92201c4ba1e20e38. 2024-12-02T17:15:53,377 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 76faba7cbca7c506fec4bb6f534477a2, disabling compactions & flushes 2024-12-02T17:15:53,377 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:53,377 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:53,377 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. after waiting 0 ms 2024-12-02T17:15:53,377 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:53,382 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/31473ece20fde70fc9741ad3ff42aadc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,382 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:53,382 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 31473ece20fde70fc9741ad3ff42aadc: Waiting for close lock at 1733159753367Running coprocessor pre-close hooks at 1733159753367Disabling compacts and flushes for region at 1733159753367Disabling writes for close at 1733159753367Writing region close event to WAL at 1733159753368 (+1 ms)Running coprocessor post-close hooks at 1733159753382 (+14 ms)Closed at 1733159753382 2024-12-02T17:15:53,383 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733159743838.31473ece20fde70fc9741ad3ff42aadc. 2024-12-02T17:15:53,383 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 51e940693c5f493c59a469d01db516d5, disabling compactions & flushes 2024-12-02T17:15:53,383 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:53,383 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:53,383 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733159743838.51e940693c5f493c59a469d01db516d5. after waiting 0 ms 2024-12-02T17:15:53,383 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:53,393 INFO [regionserver/3b3375d1a925:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T17:15:53,394 INFO [regionserver/3b3375d1a925:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T17:15:53,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741869_1045 (size=8195) 2024-12-02T17:15:53,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741869_1045 (size=8195) 2024-12-02T17:15:53,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741869_1045 (size=8195) 2024-12-02T17:15:53,406 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/info/40852d8bd0654c2cbd8ad43eacc7fa4e 2024-12-02T17:15:53,408 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/76faba7cbca7c506fec4bb6f534477a2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,410 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:53,410 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/51e940693c5f493c59a469d01db516d5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,410 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 76faba7cbca7c506fec4bb6f534477a2: Waiting for close lock at 1733159753377Running coprocessor pre-close hooks at 1733159753377Disabling compacts and flushes for region at 1733159753377Disabling writes for close at 1733159753377Writing region close event to WAL at 1733159753387 (+10 ms)Running coprocessor post-close hooks at 1733159753410 (+23 ms)Closed at 1733159753410 2024-12-02T17:15:53,411 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733159743838.76faba7cbca7c506fec4bb6f534477a2. 2024-12-02T17:15:53,411 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ba8c728daa66bac733cfa12bcd38a313, disabling compactions & flushes 2024-12-02T17:15:53,411 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:53,411 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:53,411 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. after waiting 0 ms 2024-12-02T17:15:53,411 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:53,412 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:53,412 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 51e940693c5f493c59a469d01db516d5: Waiting for close lock at 1733159753383Running coprocessor pre-close hooks at 1733159753383Disabling compacts and flushes for region at 1733159753383Disabling writes for close at 1733159753383Writing region close event to WAL at 1733159753399 (+16 ms)Running coprocessor post-close hooks at 1733159753412 (+13 ms)Closed at 1733159753412 2024-12-02T17:15:53,412 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733159743838.51e940693c5f493c59a469d01db516d5. 2024-12-02T17:15:53,412 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 16d4fe4d0593f960b6c0f8237bf0711b, disabling compactions & flushes 2024-12-02T17:15:53,412 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:53,412 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:53,412 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. after waiting 0 ms 2024-12-02T17:15:53,412 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:53,416 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/.tmp/info/40852d8bd0654c2cbd8ad43eacc7fa4e as hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/info/40852d8bd0654c2cbd8ad43eacc7fa4e 2024-12-02T17:15:53,427 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/info/40852d8bd0654c2cbd8ad43eacc7fa4e, entries=24, sequenceid=71, filesize=8.0 K 2024-12-02T17:15:53,428 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 115ms, sequenceid=71, compaction requested=false 2024-12-02T17:15:53,433 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ba8c728daa66bac733cfa12bcd38a313/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,434 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/16d4fe4d0593f960b6c0f8237bf0711b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,434 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 16d4fe4d0593f960b6c0f8237bf0711b: Waiting for close lock at 1733159753412Running coprocessor pre-close hooks at 1733159753412Disabling compacts and flushes for region at 1733159753412Disabling writes for close at 1733159753412Writing region close event to WAL at 1733159753418 (+6 ms)Running coprocessor post-close hooks at 1733159753434 (+16 ms)Closed at 1733159753434 2024-12-02T17:15:53,435 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ba8c728daa66bac733cfa12bcd38a313: Waiting for close lock at 1733159753411Running coprocessor pre-close hooks at 1733159753411Disabling compacts and flushes for region at 1733159753411Disabling writes for close at 1733159753411Writing region close event to WAL at 1733159753417 (+6 ms)Running coprocessor post-close hooks at 1733159753434 (+17 ms)Closed at 1733159753435 (+1 ms) 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1733159743838.16d4fe4d0593f960b6c0f8237bf0711b. 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733159743838.ba8c728daa66bac733cfa12bcd38a313. 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4eb95452ea2c6ad04c7114e1401424b9, disabling compactions & flushes 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eba2814ba0cf502df62df55d77023b23, disabling compactions & flushes 2024-12-02T17:15:53,435 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:53,435 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. after waiting 0 ms 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. after waiting 0 ms 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:53,435 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:53,448 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/4eb95452ea2c6ad04c7114e1401424b9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,448 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:53,448 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4eb95452ea2c6ad04c7114e1401424b9: Waiting for close lock at 1733159753435Running coprocessor pre-close hooks at 1733159753435Disabling compacts and flushes for region at 1733159753435Disabling writes for close at 1733159753435Writing region close event to WAL at 1733159753440 (+5 ms)Running coprocessor post-close hooks at 1733159753448 (+8 ms)Closed at 1733159753448 2024-12-02T17:15:53,448 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733159743838.4eb95452ea2c6ad04c7114e1401424b9. 2024-12-02T17:15:53,449 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e8a17d43fe45538f8e98a951b88f67de, disabling compactions & flushes 2024-12-02T17:15:53,449 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:53,449 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:53,449 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. after waiting 0 ms 2024-12-02T17:15:53,449 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:53,457 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/eba2814ba0cf502df62df55d77023b23/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,458 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:53,458 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eba2814ba0cf502df62df55d77023b23: Waiting for close lock at 1733159753435Running coprocessor pre-close hooks at 1733159753435Disabling compacts and flushes for region at 1733159753435Disabling writes for close at 1733159753435Writing region close event to WAL at 1733159753441 (+6 ms)Running coprocessor post-close hooks at 1733159753458 (+17 ms)Closed at 1733159753458 2024-12-02T17:15:53,458 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1733159743838.eba2814ba0cf502df62df55d77023b23. 2024-12-02T17:15:53,458 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5b72b8ea0bceb0151b39bea3aa7f2a9e, disabling compactions & flushes 2024-12-02T17:15:53,458 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:53,458 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:53,458 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. after waiting 0 ms 2024-12-02T17:15:53,459 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:53,460 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-12-02T17:15:53,461 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T17:15:53,461 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T17:15:53,461 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733159753313Running coprocessor pre-close hooks at 1733159753313Disabling compacts and flushes for region at 1733159753313Disabling writes for close at 1733159753313Obtaining lock to block concurrent updates at 1733159753313Preparing flush snapshotting stores in 1588230740 at 1733159753313Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1733159753320 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733159753327 (+7 ms)Flushing 1588230740/info: creating writer at 1733159753327Flushing 1588230740/info: appending metadata at 1733159753348 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733159753348Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30e136e4: reopening flushed file at 1733159753415 (+67 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 115ms, sequenceid=71, compaction requested=false at 1733159753428 (+13 ms)Writing region close event to WAL at 1733159753449 (+21 ms)Running coprocessor post-close hooks at 1733159753460 (+11 ms)Closed at 1733159753461 (+1 ms) 2024-12-02T17:15:53,461 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T17:15:53,465 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/e8a17d43fe45538f8e98a951b88f67de/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,466 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:53,466 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e8a17d43fe45538f8e98a951b88f67de: Waiting for close lock at 1733159753449Running coprocessor pre-close hooks at 1733159753449Disabling compacts and flushes for region at 1733159753449Disabling writes for close at 1733159753449Writing region close event to WAL at 1733159753458 (+9 ms)Running coprocessor post-close hooks at 1733159753466 (+8 ms)Closed at 1733159753466 2024-12-02T17:15:53,466 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733159743838.e8a17d43fe45538f8e98a951b88f67de. 2024-12-02T17:15:53,466 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8cd24aed41e647f8e01f1c4bdbf2c148, disabling compactions & flushes 2024-12-02T17:15:53,466 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:53,466 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:53,466 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. after waiting 0 ms 2024-12-02T17:15:53,466 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:53,469 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/5b72b8ea0bceb0151b39bea3aa7f2a9e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,470 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:53,470 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5b72b8ea0bceb0151b39bea3aa7f2a9e: Waiting for close lock at 1733159753458Running coprocessor pre-close hooks at 1733159753458Disabling compacts and flushes for region at 1733159753458Disabling writes for close at 1733159753458Writing region close event to WAL at 1733159753461 (+3 ms)Running coprocessor post-close hooks at 1733159753470 (+9 ms)Closed at 1733159753470 2024-12-02T17:15:53,470 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733159743838.5b72b8ea0bceb0151b39bea3aa7f2a9e. 2024-12-02T17:15:53,470 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d53e8d53f3fcf0c3420445811190e425, disabling compactions & flushes 2024-12-02T17:15:53,470 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:53,470 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:53,470 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. after waiting 0 ms 2024-12-02T17:15:53,470 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:53,479 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8cd24aed41e647f8e01f1c4bdbf2c148/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,480 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:53,480 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8cd24aed41e647f8e01f1c4bdbf2c148: Waiting for close lock at 1733159753466Running coprocessor pre-close hooks at 1733159753466Disabling compacts and flushes for region at 1733159753466Disabling writes for close at 1733159753466Writing region close event to WAL at 1733159753473 (+7 ms)Running coprocessor post-close hooks at 1733159753480 (+7 ms)Closed at 1733159753480 2024-12-02T17:15:53,480 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733159743838.8cd24aed41e647f8e01f1c4bdbf2c148. 2024-12-02T17:15:53,480 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ee07072225557a5ffccbd5b4dc3f8250, disabling compactions & flushes 2024-12-02T17:15:53,480 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:53,481 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:53,481 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. after waiting 0 ms 2024-12-02T17:15:53,481 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:53,482 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/d53e8d53f3fcf0c3420445811190e425/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,483 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:53,483 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d53e8d53f3fcf0c3420445811190e425: Waiting for close lock at 1733159753470Running coprocessor pre-close hooks at 1733159753470Disabling compacts and flushes for region at 1733159753470Disabling writes for close at 1733159753470Writing region close event to WAL at 1733159753476 (+6 ms)Running coprocessor post-close hooks at 1733159753483 (+7 ms)Closed at 1733159753483 2024-12-02T17:15:53,483 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733159743838.d53e8d53f3fcf0c3420445811190e425. 2024-12-02T17:15:53,483 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 80be845edde76a33d61215cef1be07d7, disabling compactions & flushes 2024-12-02T17:15:53,483 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:53,483 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:53,483 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. after waiting 0 ms 2024-12-02T17:15:53,483 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:53,490 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/ee07072225557a5ffccbd5b4dc3f8250/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,491 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:53,491 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ee07072225557a5ffccbd5b4dc3f8250: Waiting for close lock at 1733159753480Running coprocessor pre-close hooks at 1733159753480Disabling compacts and flushes for region at 1733159753480Disabling writes for close at 1733159753481 (+1 ms)Writing region close event to WAL at 1733159753481Running coprocessor post-close hooks at 1733159753491 (+10 ms)Closed at 1733159753491 2024-12-02T17:15:53,491 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733159743838.ee07072225557a5ffccbd5b4dc3f8250. 2024-12-02T17:15:53,491 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf990684b331d6bf1770501628bfa34f, disabling compactions & flushes 2024-12-02T17:15:53,491 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:53,491 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:53,491 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. after waiting 0 ms 2024-12-02T17:15:53,491 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:53,494 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/80be845edde76a33d61215cef1be07d7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,494 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:53,495 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 80be845edde76a33d61215cef1be07d7: Waiting for close lock at 1733159753483Running coprocessor pre-close hooks at 1733159753483Disabling compacts and flushes for region at 1733159753483Disabling writes for close at 1733159753483Writing region close event to WAL at 1733159753489 (+6 ms)Running coprocessor post-close hooks at 1733159753494 (+5 ms)Closed at 1733159753494 2024-12-02T17:15:53,495 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1733159743838.80be845edde76a33d61215cef1be07d7. 2024-12-02T17:15:53,495 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3882d8b2ab81cfedea2b9eda817f13bf, disabling compactions & flushes 2024-12-02T17:15:53,495 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:53,495 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:53,495 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. after waiting 0 ms 2024-12-02T17:15:53,495 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:53,505 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/cf990684b331d6bf1770501628bfa34f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,505 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:53,506 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf990684b331d6bf1770501628bfa34f: Waiting for close lock at 1733159753491Running coprocessor pre-close hooks at 1733159753491Disabling compacts and flushes for region at 1733159753491Disabling writes for close at 1733159753491Writing region close event to WAL at 1733159753492 (+1 ms)Running coprocessor post-close hooks at 1733159753505 (+13 ms)Closed at 1733159753505 2024-12-02T17:15:53,506 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733159743838.cf990684b331d6bf1770501628bfa34f. 2024-12-02T17:15:53,506 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a37d5d00fe28247586762a18d6e0943f, disabling compactions & flushes 2024-12-02T17:15:53,506 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:53,506 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:53,506 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. after waiting 0 ms 2024-12-02T17:15:53,506 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:53,508 DEBUG [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(1351): Waiting on 8981ccd8f09f082161872d7b109756f0, a37d5d00fe28247586762a18d6e0943f 2024-12-02T17:15:53,509 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/3882d8b2ab81cfedea2b9eda817f13bf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,513 DEBUG [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(1351): Waiting on 3882d8b2ab81cfedea2b9eda817f13bf, 846f82385af02c9a3c05149f7ca40c3a, bf8dc4ff69f7afd79a39737b7174af90 2024-12-02T17:15:53,514 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:53,514 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3882d8b2ab81cfedea2b9eda817f13bf: Waiting for close lock at 1733159753495Running coprocessor pre-close hooks at 1733159753495Disabling compacts and flushes for region at 1733159753495Disabling writes for close at 1733159753495Writing region close event to WAL at 1733159753496 (+1 ms)Running coprocessor post-close hooks at 1733159753513 (+17 ms)Closed at 1733159753513 2024-12-02T17:15:53,514 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1733159743838.3882d8b2ab81cfedea2b9eda817f13bf. 2024-12-02T17:15:53,514 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 846f82385af02c9a3c05149f7ca40c3a, disabling compactions & flushes 2024-12-02T17:15:53,514 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:53,514 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:53,514 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. after waiting 0 ms 2024-12-02T17:15:53,514 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:53,516 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/a37d5d00fe28247586762a18d6e0943f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,517 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:53,517 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a37d5d00fe28247586762a18d6e0943f: Waiting for close lock at 1733159753506Running coprocessor pre-close hooks at 1733159753506Disabling compacts and flushes for region at 1733159753506Disabling writes for close at 1733159753506Writing region close event to WAL at 1733159753509 (+3 ms)Running coprocessor post-close hooks at 1733159753517 (+8 ms)Closed at 1733159753517 2024-12-02T17:15:53,517 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733159743838.a37d5d00fe28247586762a18d6e0943f. 2024-12-02T17:15:53,517 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8981ccd8f09f082161872d7b109756f0, disabling compactions & flushes 2024-12-02T17:15:53,517 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:53,517 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:53,517 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. after waiting 0 ms 2024-12-02T17:15:53,517 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:53,532 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/846f82385af02c9a3c05149f7ca40c3a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,533 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:53,533 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 846f82385af02c9a3c05149f7ca40c3a: Waiting for close lock at 1733159753514Running coprocessor pre-close hooks at 1733159753514Disabling compacts and flushes for region at 1733159753514Disabling writes for close at 1733159753514Writing region close event to WAL at 1733159753514Running coprocessor post-close hooks at 1733159753533 (+19 ms)Closed at 1733159753533 2024-12-02T17:15:53,534 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733159743838.846f82385af02c9a3c05149f7ca40c3a. 2024-12-02T17:15:53,534 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bf8dc4ff69f7afd79a39737b7174af90, disabling compactions & flushes 2024-12-02T17:15:53,534 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:53,534 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:53,534 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. after waiting 0 ms 2024-12-02T17:15:53,534 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:53,537 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/8981ccd8f09f082161872d7b109756f0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,538 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:53,538 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8981ccd8f09f082161872d7b109756f0: Waiting for close lock at 1733159753517Running coprocessor pre-close hooks at 1733159753517Disabling compacts and flushes for region at 1733159753517Disabling writes for close at 1733159753517Writing region close event to WAL at 1733159753520 (+3 ms)Running coprocessor post-close hooks at 1733159753538 (+18 ms)Closed at 1733159753538 2024-12-02T17:15:53,538 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733159743838.8981ccd8f09f082161872d7b109756f0. 2024-12-02T17:15:53,552 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/data/default/test/bf8dc4ff69f7afd79a39737b7174af90/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:53,553 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:53,553 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bf8dc4ff69f7afd79a39737b7174af90: Waiting for close lock at 1733159753534Running coprocessor pre-close hooks at 1733159753534Disabling compacts and flushes for region at 1733159753534Disabling writes for close at 1733159753534Writing region close event to WAL at 1733159753540 (+6 ms)Running coprocessor post-close hooks at 1733159753553 (+13 ms)Closed at 1733159753553 2024-12-02T17:15:53,553 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1733159743838.bf8dc4ff69f7afd79a39737b7174af90. 2024-12-02T17:15:53,708 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(976): stopping server 3b3375d1a925,34079,1733159740078; all regions closed. 2024-12-02T17:15:53,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741833_1009 (size=7064) 2024-12-02T17:15:53,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741833_1009 (size=7064) 2024-12-02T17:15:53,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741833_1009 (size=7064) 2024-12-02T17:15:53,713 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(976): stopping server 3b3375d1a925,33117,1733159739837; all regions closed. 2024-12-02T17:15:53,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741865_1041 (size=4204) 2024-12-02T17:15:53,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741865_1041 (size=4204) 2024-12-02T17:15:53,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741865_1041 (size=4204) 2024-12-02T17:15:53,717 DEBUG [RS:2;3b3375d1a925:34079 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs 2024-12-02T17:15:53,718 INFO [RS:2;3b3375d1a925:34079 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3b3375d1a925%2C34079%2C1733159740078:(num 1733159742528) 2024-12-02T17:15:53,718 DEBUG [RS:2;3b3375d1a925:34079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:53,718 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T17:15:53,718 INFO [RS:2;3b3375d1a925:34079 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T17:15:53,718 INFO [RS:2;3b3375d1a925:34079 {}] hbase.ChoreService(370): Chore service for: regionserver/3b3375d1a925:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T17:15:53,719 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T17:15:53,719 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T17:15:53,719 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T17:15:53,719 INFO [RS:2;3b3375d1a925:34079 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T17:15:53,719 INFO [RS:2;3b3375d1a925:34079 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34079 2024-12-02T17:15:53,719 INFO [regionserver/3b3375d1a925:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T17:15:53,721 DEBUG [RS:1;3b3375d1a925:33117 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs 2024-12-02T17:15:53,722 INFO [RS:1;3b3375d1a925:33117 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3b3375d1a925%2C33117%2C1733159739837.meta:.meta(num 1733159749707) 2024-12-02T17:15:53,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T17:15:53,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3b3375d1a925,34079,1733159740078 2024-12-02T17:15:53,723 INFO [RS:2;3b3375d1a925:34079 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T17:15:53,723 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3b3375d1a925,34079,1733159740078] 2024-12-02T17:15:53,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741834_1010 (size=7074) 2024-12-02T17:15:53,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741834_1010 (size=7074) 2024-12-02T17:15:53,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741834_1010 (size=7074) 2024-12-02T17:15:53,726 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3b3375d1a925,34079,1733159740078 already deleted, retry=false 2024-12-02T17:15:53,726 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3b3375d1a925,34079,1733159740078 expired; onlineServers=1 2024-12-02T17:15:53,727 DEBUG [RS:1;3b3375d1a925:33117 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/oldWALs 2024-12-02T17:15:53,727 INFO [RS:1;3b3375d1a925:33117 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3b3375d1a925%2C33117%2C1733159739837:(num 1733159742606) 2024-12-02T17:15:53,727 DEBUG [RS:1;3b3375d1a925:33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:53,728 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T17:15:53,728 INFO [RS:1;3b3375d1a925:33117 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T17:15:53,728 INFO [RS:1;3b3375d1a925:33117 {}] hbase.ChoreService(370): Chore service for: regionserver/3b3375d1a925:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T17:15:53,728 INFO [RS:1;3b3375d1a925:33117 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T17:15:53,728 INFO [regionserver/3b3375d1a925:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T17:15:53,728 INFO [RS:1;3b3375d1a925:33117 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33117 2024-12-02T17:15:53,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T17:15:53,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3b3375d1a925,33117,1733159739837 2024-12-02T17:15:53,730 INFO [RS:1;3b3375d1a925:33117 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T17:15:53,732 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3b3375d1a925,33117,1733159739837] 2024-12-02T17:15:53,733 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3b3375d1a925,33117,1733159739837 already deleted, retry=false 2024-12-02T17:15:53,733 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3b3375d1a925,33117,1733159739837 expired; onlineServers=0 2024-12-02T17:15:53,733 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3b3375d1a925,37351,1733159738385' ***** 2024-12-02T17:15:53,734 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T17:15:53,734 INFO [M:0;3b3375d1a925:37351 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T17:15:53,734 INFO [M:0;3b3375d1a925:37351 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T17:15:53,734 DEBUG [M:0;3b3375d1a925:37351 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T17:15:53,734 DEBUG [M:0;3b3375d1a925:37351 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T17:15:53,734 DEBUG [master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.large.0-1733159741990 {}] cleaner.HFileCleaner(306): Exit Thread[master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.large.0-1733159741990,5,FailOnTimeoutGroup] 2024-12-02T17:15:53,734 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T17:15:53,734 DEBUG [master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.small.0-1733159741997 {}] cleaner.HFileCleaner(306): Exit Thread[master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.small.0-1733159741997,5,FailOnTimeoutGroup] 2024-12-02T17:15:53,736 INFO [M:0;3b3375d1a925:37351 {}] hbase.ChoreService(370): Chore service for: master/3b3375d1a925:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T17:15:53,736 INFO [M:0;3b3375d1a925:37351 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T17:15:53,736 DEBUG [M:0;3b3375d1a925:37351 {}] master.HMaster(1795): Stopping service threads 2024-12-02T17:15:53,736 INFO [M:0;3b3375d1a925:37351 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T17:15:53,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T17:15:53,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:15:53,736 INFO [M:0;3b3375d1a925:37351 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:15:53,736 INFO [M:0;3b3375d1a925:37351 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:15:53,736 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T17:15:53,737 DEBUG [M:0;3b3375d1a925:37351 {}] zookeeper.ZKUtil(347): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T17:15:53,737 WARN [M:0;3b3375d1a925:37351 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T17:15:53,738 INFO [M:0;3b3375d1a925:37351 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/.lastflushedseqids 2024-12-02T17:15:53,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741870_1046 (size=1832) 2024-12-02T17:15:53,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741870_1046 (size=1832) 2024-12-02T17:15:53,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741870_1046 (size=1832) 2024-12-02T17:15:53,760 INFO [M:0;3b3375d1a925:37351 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:15:53,760 INFO [M:0;3b3375d1a925:37351 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T17:15:53,761 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T17:15:53,761 INFO [M:0;3b3375d1a925:37351 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:15:53,761 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:15:53,761 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T17:15:53,761 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:15:53,761 INFO [M:0;3b3375d1a925:37351 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.10 KB heapSize=301.48 KB 2024-12-02T17:15:53,786 DEBUG [M:0;3b3375d1a925:37351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0f8f3d65b0b240838f64087c5b7989a0 is 82, key is hbase:meta,,1/info:regioninfo/1733159750777/Put/seqid=0 2024-12-02T17:15:53,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741871_1047 (size=6071) 2024-12-02T17:15:53,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741871_1047 (size=6071) 2024-12-02T17:15:53,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741871_1047 (size=6071) 2024-12-02T17:15:53,805 INFO [M:0;3b3375d1a925:37351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0f8f3d65b0b240838f64087c5b7989a0 2024-12-02T17:15:53,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:15:53,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34079-0x1009bfce1af0003, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:15:53,826 INFO [RS:2;3b3375d1a925:34079 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T17:15:53,826 INFO [RS:2;3b3375d1a925:34079 {}] regionserver.HRegionServer(1031): Exiting; stopping=3b3375d1a925,34079,1733159740078; zookeeper connection closed. 2024-12-02T17:15:53,827 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@56059723 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@56059723 2024-12-02T17:15:53,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:15:53,832 INFO [RS:1;3b3375d1a925:33117 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T17:15:53,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33117-0x1009bfce1af0002, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:15:53,832 INFO [RS:1;3b3375d1a925:33117 {}] regionserver.HRegionServer(1031): Exiting; stopping=3b3375d1a925,33117,1733159739837; zookeeper connection closed. 2024-12-02T17:15:53,838 DEBUG [M:0;3b3375d1a925:37351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1cd024cc05334c4f8c7e8e8f4d1488bd is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1733159753270/Put/seqid=0 2024-12-02T17:15:53,839 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@440957ca {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@440957ca 2024-12-02T17:15:53,840 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-02T17:15:53,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741872_1048 (size=12801) 2024-12-02T17:15:53,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741872_1048 (size=12801) 2024-12-02T17:15:53,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741872_1048 (size=12801) 2024-12-02T17:15:53,865 INFO [M:0;3b3375d1a925:37351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=246.83 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1cd024cc05334c4f8c7e8e8f4d1488bd 2024-12-02T17:15:53,891 DEBUG [M:0;3b3375d1a925:37351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f5ab8de0ecdb4a63bb9156a319b7ac0f is 69, key is 3b3375d1a925,33117,1733159739837/rs:state/1733159742122/Put/seqid=0 2024-12-02T17:15:53,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741873_1049 (size=5442) 2024-12-02T17:15:53,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741873_1049 (size=5442) 2024-12-02T17:15:53,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741873_1049 (size=5442) 2024-12-02T17:15:53,906 INFO [M:0;3b3375d1a925:37351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f5ab8de0ecdb4a63bb9156a319b7ac0f 2024-12-02T17:15:53,912 INFO [M:0;3b3375d1a925:37351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f5ab8de0ecdb4a63bb9156a319b7ac0f 2024-12-02T17:15:53,930 DEBUG [M:0;3b3375d1a925:37351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d1422f57f0c245bc86a640280ed03f59 is 52, key is load_balancer_on/state:d/1733159748517/Put/seqid=0 2024-12-02T17:15:53,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741874_1050 (size=5058) 2024-12-02T17:15:53,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741874_1050 (size=5058) 2024-12-02T17:15:53,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741874_1050 (size=5058) 2024-12-02T17:15:53,949 INFO [M:0;3b3375d1a925:37351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d1422f57f0c245bc86a640280ed03f59 2024-12-02T17:15:53,962 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0f8f3d65b0b240838f64087c5b7989a0 as hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0f8f3d65b0b240838f64087c5b7989a0 2024-12-02T17:15:53,970 INFO [M:0;3b3375d1a925:37351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0f8f3d65b0b240838f64087c5b7989a0, entries=14, sequenceid=569, filesize=5.9 K 2024-12-02T17:15:53,972 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1cd024cc05334c4f8c7e8e8f4d1488bd as hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1cd024cc05334c4f8c7e8e8f4d1488bd 2024-12-02T17:15:53,982 INFO [M:0;3b3375d1a925:37351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1cd024cc05334c4f8c7e8e8f4d1488bd, entries=79, sequenceid=569, filesize=12.5 K 2024-12-02T17:15:53,983 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f5ab8de0ecdb4a63bb9156a319b7ac0f as hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f5ab8de0ecdb4a63bb9156a319b7ac0f 2024-12-02T17:15:53,991 INFO [M:0;3b3375d1a925:37351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f5ab8de0ecdb4a63bb9156a319b7ac0f 2024-12-02T17:15:53,992 INFO [M:0;3b3375d1a925:37351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f5ab8de0ecdb4a63bb9156a319b7ac0f, entries=3, sequenceid=569, filesize=5.3 K 2024-12-02T17:15:53,994 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d1422f57f0c245bc86a640280ed03f59 as hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d1422f57f0c245bc86a640280ed03f59 2024-12-02T17:15:54,003 INFO [M:0;3b3375d1a925:37351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/b809de5e-83cc-a9d5-5087-eaafd409f567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d1422f57f0c245bc86a640280ed03f59, entries=1, sequenceid=569, filesize=4.9 K 2024-12-02T17:15:54,004 INFO [M:0;3b3375d1a925:37351 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.10 KB/254054, heapSize ~301.42 KB/308656, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 243ms, sequenceid=569, compaction requested=false 2024-12-02T17:15:54,006 INFO [M:0;3b3375d1a925:37351 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:15:54,006 DEBUG [M:0;3b3375d1a925:37351 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733159753761Disabling compacts and flushes for region at 1733159753761Disabling writes for close at 1733159753761Obtaining lock to block concurrent updates at 1733159753761Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733159753761Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254054, getHeapSize=308656, getOffHeapSize=0, getCellsCount=708 at 1733159753762 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733159753762Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733159753762Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733159753785 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733159753785Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733159753812 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733159753837 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733159753837Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733159753873 (+36 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733159753890 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733159753890Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733159753913 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733159753929 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733159753930 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ea07c5c: reopening flushed file at 1733159753960 (+30 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d7e7ff9: reopening flushed file at 1733159753971 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54652a05: reopening flushed file at 1733159753982 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3585e693: reopening flushed file at 1733159753992 (+10 ms)Finished flush of dataSize ~248.10 KB/254054, heapSize ~301.42 KB/308656, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 243ms, sequenceid=569, compaction requested=false at 1733159754004 (+12 ms)Writing region close event to WAL at 1733159754006 (+2 ms)Closed at 1733159754006 2024-12-02T17:15:54,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41165 is added to blk_1073741830_1006 (size=292048) 2024-12-02T17:15:54,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40475 is added to blk_1073741830_1006 (size=292048) 2024-12-02T17:15:54,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38149 is added to blk_1073741830_1006 (size=292048) 2024-12-02T17:15:54,015 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T17:15:54,015 INFO [M:0;3b3375d1a925:37351 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T17:15:54,015 INFO [M:0;3b3375d1a925:37351 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37351 2024-12-02T17:15:54,015 INFO [M:0;3b3375d1a925:37351 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T17:15:54,118 INFO [M:0;3b3375d1a925:37351 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T17:15:54,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:15:54,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37351-0x1009bfce1af0000, quorum=127.0.0.1:60644, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:15:54,122 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67ddfd25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T17:15:54,124 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T17:15:54,124 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T17:15:54,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T17:15:54,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.log.dir/,STOPPED} 2024-12-02T17:15:54,127 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T17:15:54,127 WARN [BP-1138772532-172.17.0.3-1733159732851 heartbeating to localhost/127.0.0.1:43373 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T17:15:54,127 WARN [BP-1138772532-172.17.0.3-1733159732851 heartbeating to localhost/127.0.0.1:43373 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1138772532-172.17.0.3-1733159732851 (Datanode Uuid b3255ec2-de5d-4782-8cae-b92c55db2949) service to localhost/127.0.0.1:43373 2024-12-02T17:15:54,127 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T17:15:54,128 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data5/current/BP-1138772532-172.17.0.3-1733159732851 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T17:15:54,128 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data6/current/BP-1138772532-172.17.0.3-1733159732851 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T17:15:54,129 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T17:15:54,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cfccf27{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T17:15:54,131 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T17:15:54,131 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T17:15:54,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T17:15:54,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.log.dir/,STOPPED} 2024-12-02T17:15:54,133 WARN [BP-1138772532-172.17.0.3-1733159732851 heartbeating to localhost/127.0.0.1:43373 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T17:15:54,133 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T17:15:54,133 WARN [BP-1138772532-172.17.0.3-1733159732851 heartbeating to localhost/127.0.0.1:43373 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1138772532-172.17.0.3-1733159732851 (Datanode Uuid d84305b4-2c40-456c-81a0-6f632adccdf2) service to localhost/127.0.0.1:43373 2024-12-02T17:15:54,133 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T17:15:54,133 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data3/current/BP-1138772532-172.17.0.3-1733159732851 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T17:15:54,133 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T17:15:54,133 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data4/current/BP-1138772532-172.17.0.3-1733159732851 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T17:15:54,135 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2226d6a6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T17:15:54,136 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T17:15:54,136 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T17:15:54,136 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T17:15:54,136 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.log.dir/,STOPPED} 2024-12-02T17:15:54,137 WARN [BP-1138772532-172.17.0.3-1733159732851 heartbeating to localhost/127.0.0.1:43373 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T17:15:54,137 WARN [BP-1138772532-172.17.0.3-1733159732851 heartbeating to localhost/127.0.0.1:43373 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1138772532-172.17.0.3-1733159732851 (Datanode Uuid bc6d0b55-28fe-4b0b-b2ec-d095bd55a2af) service to localhost/127.0.0.1:43373 2024-12-02T17:15:54,137 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T17:15:54,137 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T17:15:54,138 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data1/current/BP-1138772532-172.17.0.3-1733159732851 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T17:15:54,138 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/cluster_34a968f3-04d7-8643-2247-f8ea9fdc0841/data/data2/current/BP-1138772532-172.17.0.3-1733159732851 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T17:15:54,138 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T17:15:54,146 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@376fa8f3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T17:15:54,147 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T17:15:54,147 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T17:15:54,147 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T17:15:54,148 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1e388feb-7450-4178-437c-c20912783c48/hadoop.log.dir/,STOPPED} 2024-12-02T17:15:54,160 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T17:15:54,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down