2024-11-27 01:14:25,651 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@65f8f5ae 2024-11-27 01:14:25,674 main DEBUG Took 0.020248 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 01:14:25,675 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 01:14:25,675 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 01:14:25,676 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 01:14:25,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,700 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 01:14:25,717 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,719 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,720 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,721 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,722 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,722 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,723 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,723 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,724 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,725 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,726 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,726 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,727 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,728 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,729 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,729 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,730 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:14:25,731 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,732 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 01:14:25,734 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:14:25,735 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 01:14:25,738 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 01:14:25,738 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 01:14:25,740 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 01:14:25,740 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 01:14:25,752 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 01:14:25,756 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 01:14:25,758 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 01:14:25,758 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 01:14:25,759 main DEBUG createAppenders(={Console}) 2024-11-27 01:14:25,760 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@65f8f5ae initialized 2024-11-27 01:14:25,760 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@65f8f5ae 2024-11-27 01:14:25,761 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@65f8f5ae OK. 2024-11-27 01:14:25,761 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 01:14:25,762 main DEBUG OutputStream closed 2024-11-27 01:14:25,762 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 01:14:25,763 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 01:14:25,763 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@485966cc OK 2024-11-27 01:14:25,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 01:14:25,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 01:14:25,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 01:14:25,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 01:14:25,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 01:14:25,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 01:14:25,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 01:14:25,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 01:14:25,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 01:14:25,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 01:14:25,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 01:14:25,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 01:14:25,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 01:14:25,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 01:14:25,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 01:14:25,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 01:14:25,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 01:14:25,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 01:14:25,914 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 01:14:25,921 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39a2bb97) with optional ClassLoader: null 2024-11-27 01:14:25,922 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 01:14:25,923 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39a2bb97] started OK. 2024-11-27T01:14:25,946 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-27 01:14:25,951 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 01:14:25,951 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T01:14:26,636 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1 2024-11-27T01:14:26,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T01:14:26,743 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2, deleteOnExit=true 2024-11-27T01:14:26,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T01:14:26,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/test.cache.data in system properties and HBase conf 2024-11-27T01:14:26,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T01:14:26,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.log.dir in system properties and HBase conf 2024-11-27T01:14:26,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T01:14:26,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T01:14:26,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T01:14:26,864 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T01:14:26,990 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T01:14:26,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T01:14:26,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T01:14:26,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T01:14:26,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T01:14:26,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T01:14:26,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T01:14:26,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T01:14:27,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T01:14:27,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T01:14:27,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/nfs.dump.dir in system properties and HBase conf 2024-11-27T01:14:27,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/java.io.tmpdir in system properties and HBase conf 2024-11-27T01:14:27,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T01:14:27,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T01:14:27,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T01:14:28,426 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T01:14:28,603 INFO [Time-limited test {}] log.Log(170): Logging initialized @4233ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T01:14:28,799 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:14:28,917 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:14:28,960 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:14:28,961 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:14:28,962 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T01:14:28,985 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:14:28,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@952689a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:14:28,990 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@378c34a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:14:29,253 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@302cfe51{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/java.io.tmpdir/jetty-localhost-44279-hadoop-hdfs-3_4_1-tests_jar-_-any-15761745628332477465/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T01:14:29,264 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c1eb5a9{HTTP/1.1, (http/1.1)}{localhost:44279} 2024-11-27T01:14:29,265 INFO [Time-limited test {}] server.Server(415): Started @4897ms 2024-11-27T01:14:29,869 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:14:29,885 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:14:29,889 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:14:29,889 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:14:29,889 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T01:14:29,892 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18a3047c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:14:29,897 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62c0eab0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:14:30,073 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a0ce16d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/java.io.tmpdir/jetty-localhost-36731-hadoop-hdfs-3_4_1-tests_jar-_-any-8219878543987157156/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:14:30,074 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1cafe9ff{HTTP/1.1, (http/1.1)}{localhost:36731} 2024-11-27T01:14:30,075 INFO [Time-limited test {}] server.Server(415): Started @5707ms 2024-11-27T01:14:30,155 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:14:30,351 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:14:30,358 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:14:30,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:14:30,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:14:30,361 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T01:14:30,362 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a294b42{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:14:30,363 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b427afd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:14:30,533 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2dca9824{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/java.io.tmpdir/jetty-localhost-46753-hadoop-hdfs-3_4_1-tests_jar-_-any-3030690491902137097/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:14:30,534 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@46b93c21{HTTP/1.1, (http/1.1)}{localhost:46753} 2024-11-27T01:14:30,535 INFO [Time-limited test {}] server.Server(415): Started @6167ms 2024-11-27T01:14:30,539 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:14:30,677 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:14:30,691 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:14:30,721 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:14:30,721 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:14:30,721 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T01:14:30,730 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@609b663f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:14:30,731 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a0c767{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:14:30,874 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data1/current/BP-1607804123-172.17.0.2-1732670067970/current, will proceed with Du for space computation calculation, 2024-11-27T01:14:30,876 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data3/current/BP-1607804123-172.17.0.2-1732670067970/current, will proceed with Du for space computation calculation, 2024-11-27T01:14:30,878 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data2/current/BP-1607804123-172.17.0.2-1732670067970/current, will proceed with Du for space computation calculation, 2024-11-27T01:14:30,882 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data4/current/BP-1607804123-172.17.0.2-1732670067970/current, will proceed with Du for space computation calculation, 2024-11-27T01:14:30,948 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3351f4c8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/java.io.tmpdir/jetty-localhost-44077-hadoop-hdfs-3_4_1-tests_jar-_-any-7226491072175090105/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:14:30,952 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51924127{HTTP/1.1, (http/1.1)}{localhost:44077} 2024-11-27T01:14:30,952 INFO [Time-limited test {}] server.Server(415): Started @6584ms 2024-11-27T01:14:30,956 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:14:30,970 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:14:30,976 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:14:31,061 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5c649538f14dc0e1 with lease ID 0xbe69b6e2ee5e54b9: Processing first storage report for DS-34aa0431-5d12-49f8-a64c-c90b8fe9f878 from datanode DatanodeRegistration(127.0.0.1:39429, datanodeUuid=02f1b4ec-72df-4ada-929d-8dd81b7b18e0, infoPort=34837, infoSecurePort=0, ipcPort=33657, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970) 2024-11-27T01:14:31,062 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5c649538f14dc0e1 with lease ID 0xbe69b6e2ee5e54b9: from storage DS-34aa0431-5d12-49f8-a64c-c90b8fe9f878 node DatanodeRegistration(127.0.0.1:39429, datanodeUuid=02f1b4ec-72df-4ada-929d-8dd81b7b18e0, infoPort=34837, infoSecurePort=0, ipcPort=33657, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-27T01:14:31,063 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5c649538f14dc0e1 with lease ID 0xbe69b6e2ee5e54b9: Processing first storage report for DS-0a633a01-4998-4b1d-b410-c65187a9932e from datanode DatanodeRegistration(127.0.0.1:39429, datanodeUuid=02f1b4ec-72df-4ada-929d-8dd81b7b18e0, infoPort=34837, infoSecurePort=0, ipcPort=33657, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970) 2024-11-27T01:14:31,063 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5c649538f14dc0e1 with lease ID 0xbe69b6e2ee5e54b9: from storage DS-0a633a01-4998-4b1d-b410-c65187a9932e node DatanodeRegistration(127.0.0.1:39429, datanodeUuid=02f1b4ec-72df-4ada-929d-8dd81b7b18e0, infoPort=34837, infoSecurePort=0, ipcPort=33657, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:14:31,065 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x55d9b9a9e760666c with lease ID 0xbe69b6e2ee5e54ba: Processing first storage report for DS-868d7a4e-f180-4a29-adfe-c9198fcd7493 from datanode DatanodeRegistration(127.0.0.1:46391, datanodeUuid=7ef4efb0-4b93-4b26-aae3-73e7ee13a4f2, infoPort=42521, infoSecurePort=0, ipcPort=37421, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970) 2024-11-27T01:14:31,066 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x55d9b9a9e760666c with lease ID 0xbe69b6e2ee5e54ba: from storage DS-868d7a4e-f180-4a29-adfe-c9198fcd7493 node DatanodeRegistration(127.0.0.1:46391, datanodeUuid=7ef4efb0-4b93-4b26-aae3-73e7ee13a4f2, infoPort=42521, infoSecurePort=0, ipcPort=37421, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:14:31,066 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x55d9b9a9e760666c with lease ID 0xbe69b6e2ee5e54ba: Processing first storage report for DS-6bf99c16-e6a2-4079-958e-cbad5b43bd9d from datanode DatanodeRegistration(127.0.0.1:46391, datanodeUuid=7ef4efb0-4b93-4b26-aae3-73e7ee13a4f2, infoPort=42521, infoSecurePort=0, ipcPort=37421, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970) 2024-11-27T01:14:31,066 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x55d9b9a9e760666c with lease ID 0xbe69b6e2ee5e54ba: from storage DS-6bf99c16-e6a2-4079-958e-cbad5b43bd9d node DatanodeRegistration(127.0.0.1:46391, datanodeUuid=7ef4efb0-4b93-4b26-aae3-73e7ee13a4f2, infoPort=42521, infoSecurePort=0, ipcPort=37421, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:14:31,221 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data5/current/BP-1607804123-172.17.0.2-1732670067970/current, will proceed with Du for space computation calculation, 2024-11-27T01:14:31,224 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data6/current/BP-1607804123-172.17.0.2-1732670067970/current, will proceed with Du for space computation calculation, 2024-11-27T01:14:31,337 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:14:31,347 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x716520ad8e5fbfdc with lease ID 0xbe69b6e2ee5e54bb: Processing first storage report for DS-ff782df8-ea5f-46cc-9d82-6b5a6f27a76c from datanode DatanodeRegistration(127.0.0.1:40703, datanodeUuid=2b23d955-eed6-4dac-bb99-b526474368cc, infoPort=34009, infoSecurePort=0, ipcPort=37337, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970) 2024-11-27T01:14:31,348 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x716520ad8e5fbfdc with lease ID 0xbe69b6e2ee5e54bb: from storage DS-ff782df8-ea5f-46cc-9d82-6b5a6f27a76c node DatanodeRegistration(127.0.0.1:40703, datanodeUuid=2b23d955-eed6-4dac-bb99-b526474368cc, infoPort=34009, infoSecurePort=0, ipcPort=37337, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T01:14:31,348 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x716520ad8e5fbfdc with lease ID 0xbe69b6e2ee5e54bb: Processing first storage report for DS-2975b8ee-c029-4d57-b613-8e19aba1689d from datanode DatanodeRegistration(127.0.0.1:40703, datanodeUuid=2b23d955-eed6-4dac-bb99-b526474368cc, infoPort=34009, infoSecurePort=0, ipcPort=37337, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970) 2024-11-27T01:14:31,348 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x716520ad8e5fbfdc with lease ID 0xbe69b6e2ee5e54bb: from storage DS-2975b8ee-c029-4d57-b613-8e19aba1689d node DatanodeRegistration(127.0.0.1:40703, datanodeUuid=2b23d955-eed6-4dac-bb99-b526474368cc, infoPort=34009, infoSecurePort=0, ipcPort=37337, storageInfo=lv=-57;cid=testClusterID;nsid=47035423;c=1732670067970), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:14:31,566 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1 2024-11-27T01:14:31,658 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/zookeeper_0, clientPort=54756, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T01:14:31,674 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54756 2024-11-27T01:14:31,691 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:31,695 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:32,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:14:32,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:14:32,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:14:32,500 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d with version=8 2024-11-27T01:14:32,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/hbase-staging 2024-11-27T01:14:32,604 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T01:14:32,926 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:14:32,941 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:32,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:32,948 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:14:32,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:32,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:14:33,189 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T01:14:33,268 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T01:14:33,280 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T01:14:33,285 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:14:33,322 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7578 (auto-detected) 2024-11-27T01:14:33,323 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T01:14:33,352 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33239 2024-11-27T01:14:33,380 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33239 connecting to ZooKeeper ensemble=127.0.0.1:54756 2024-11-27T01:14:33,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:332390x0, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:14:33,426 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33239-0x10156fd0ddf0000 connected 2024-11-27T01:14:33,479 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:33,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:33,500 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:14:33,506 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d, hbase.cluster.distributed=false 2024-11-27T01:14:33,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:14:33,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33239 2024-11-27T01:14:33,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33239 2024-11-27T01:14:33,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33239 2024-11-27T01:14:33,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33239 2024-11-27T01:14:33,593 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33239 2024-11-27T01:14:33,737 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:14:33,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:33,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:33,740 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:14:33,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:33,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:14:33,743 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:14:33,746 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:14:33,747 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39915 2024-11-27T01:14:33,749 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39915 connecting to ZooKeeper ensemble=127.0.0.1:54756 2024-11-27T01:14:33,750 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:33,753 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:33,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399150x0, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:14:33,764 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:399150x0, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:14:33,768 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39915-0x10156fd0ddf0001 connected 2024-11-27T01:14:33,769 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:14:33,780 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:14:33,783 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:14:33,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:14:33,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39915 2024-11-27T01:14:33,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39915 2024-11-27T01:14:33,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39915 2024-11-27T01:14:33,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39915 2024-11-27T01:14:33,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39915 2024-11-27T01:14:33,826 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:14:33,826 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:33,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:33,827 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:14:33,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:33,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:14:33,828 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:14:33,828 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:14:33,829 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40647 2024-11-27T01:14:33,832 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40647 connecting to ZooKeeper ensemble=127.0.0.1:54756 2024-11-27T01:14:33,833 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:33,837 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:33,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406470x0, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:14:33,845 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:406470x0, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:14:33,845 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40647-0x10156fd0ddf0002 connected 2024-11-27T01:14:33,845 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:14:33,847 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:14:33,848 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:14:33,852 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:14:33,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40647 2024-11-27T01:14:33,857 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40647 2024-11-27T01:14:33,857 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40647 2024-11-27T01:14:33,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40647 2024-11-27T01:14:33,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40647 2024-11-27T01:14:33,882 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:14:33,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:33,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:33,883 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:14:33,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:14:33,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:14:33,884 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:14:33,884 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:14:33,891 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32943 2024-11-27T01:14:33,893 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32943 connecting to ZooKeeper ensemble=127.0.0.1:54756 2024-11-27T01:14:33,895 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:33,898 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:33,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329430x0, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:14:33,909 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:14:33,910 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:14:33,909 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32943-0x10156fd0ddf0003 connected 2024-11-27T01:14:33,911 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:14:33,913 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:14:33,915 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:14:33,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32943 2024-11-27T01:14:33,921 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32943 2024-11-27T01:14:33,922 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32943 2024-11-27T01:14:33,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32943 2024-11-27T01:14:33,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32943 2024-11-27T01:14:33,945 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7d6f26a4c136:33239 2024-11-27T01:14:33,946 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:33,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:14:33,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:14:33,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:14:33,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:14:33,964 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:33,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:14:33,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:14:33,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:33,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:33,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:14:33,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:33,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:33,991 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T01:14:33,992 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7d6f26a4c136,33239,1732670072670 from backup master directory 2024-11-27T01:14:33,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:14:33,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:14:33,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:14:33,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:33,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:14:33,999 WARN [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:14:34,000 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:34,002 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T01:14:34,004 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T01:14:34,088 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/hbase.id] with ID: 0b70014f-cd86-42c2-939d-a21e25f574a1 2024-11-27T01:14:34,088 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/.tmp/hbase.id 2024-11-27T01:14:34,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:14:34,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:14:34,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:14:34,161 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/.tmp/hbase.id]:[hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/hbase.id] 2024-11-27T01:14:34,252 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:14:34,266 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T01:14:34,310 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 29ms. 2024-11-27T01:14:34,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:34,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:34,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:34,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:34,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:14:34,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:14:34,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:14:34,424 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:14:34,427 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T01:14:34,454 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:14:34,460 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:14:34,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:14:34,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:14:34,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:14:34,530 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store 2024-11-27T01:14:34,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:14:34,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:14:34,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:14:34,565 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T01:14:34,569 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:34,570 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T01:14:34,570 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:14:34,571 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:14:34,572 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T01:14:34,573 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:14:34,573 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:14:34,574 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732670074570Disabling compacts and flushes for region at 1732670074570Disabling writes for close at 1732670074573 (+3 ms)Writing region close event to WAL at 1732670074573Closed at 1732670074573 2024-11-27T01:14:34,576 WARN [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/.initializing 2024-11-27T01:14:34,576 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/WALs/7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:34,585 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:14:34,605 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C33239%2C1732670072670, suffix=, logDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/WALs/7d6f26a4c136,33239,1732670072670, archiveDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/oldWALs, maxLogs=10 2024-11-27T01:14:34,635 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/WALs/7d6f26a4c136,33239,1732670072670/7d6f26a4c136%2C33239%2C1732670072670.1732670074611, exclude list is [], retry=0 2024-11-27T01:14:34,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46391,DS-868d7a4e-f180-4a29-adfe-c9198fcd7493,DISK] 2024-11-27T01:14:34,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39429,DS-34aa0431-5d12-49f8-a64c-c90b8fe9f878,DISK] 2024-11-27T01:14:34,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40703,DS-ff782df8-ea5f-46cc-9d82-6b5a6f27a76c,DISK] 2024-11-27T01:14:34,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T01:14:34,712 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/WALs/7d6f26a4c136,33239,1732670072670/7d6f26a4c136%2C33239%2C1732670072670.1732670074611 2024-11-27T01:14:34,713 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34009:34009),(127.0.0.1/127.0.0.1:34837:34837),(127.0.0.1/127.0.0.1:42521:42521)] 2024-11-27T01:14:34,714 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:14:34,715 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:34,719 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,720 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,769 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T01:14:34,809 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:34,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:34,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,817 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T01:14:34,817 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:34,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:34,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T01:14:34,823 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:34,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:34,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T01:14:34,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:34,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:34,830 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,834 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,835 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,840 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,841 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,845 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:14:34,849 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:14:34,858 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:34,860 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70910655, jitterRate=0.056651100516319275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:14:34,868 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732670074736Initializing all the Stores at 1732670074739 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670074739Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670074740 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670074740Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670074740Cleaning up temporary data from old regions at 1732670074841 (+101 ms)Region opened successfully at 1732670074868 (+27 ms) 2024-11-27T01:14:34,869 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T01:14:34,918 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31747fff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:14:34,958 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T01:14:34,971 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:34,971 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:34,975 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:34,976 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T01:14:34,983 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-27T01:14:34,983 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T01:14:35,020 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T01:14:35,031 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T01:14:35,033 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T01:14:35,036 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T01:14:35,037 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T01:14:35,041 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T01:14:35,044 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T01:14:35,048 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T01:14:35,050 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T01:14:35,053 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T01:14:35,054 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T01:14:35,075 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T01:14:35,077 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T01:14:35,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:14:35,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:14:35,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:14:35,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:14:35,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,089 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7d6f26a4c136,33239,1732670072670, sessionid=0x10156fd0ddf0000, setting cluster-up flag (Was=false) 2024-11-27T01:14:35,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,113 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T01:14:35,115 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:35,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:35,134 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T01:14:35,135 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:35,143 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T01:14:35,229 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer(746): ClusterId : 0b70014f-cd86-42c2-939d-a21e25f574a1 2024-11-27T01:14:35,229 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(746): ClusterId : 0b70014f-cd86-42c2-939d-a21e25f574a1 2024-11-27T01:14:35,233 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:14:35,232 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:35,234 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(746): ClusterId : 0b70014f-cd86-42c2-939d-a21e25f574a1 2024-11-27T01:14:35,234 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:14:35,239 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:14:35,246 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:14:35,246 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:14:35,247 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T01:14:35,248 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:14:35,248 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:14:35,253 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:14:35,254 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:14:35,254 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:14:35,255 DEBUG [RS:1;7d6f26a4c136:40647 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ccde7b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:14:35,256 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:14:35,260 DEBUG [RS:0;7d6f26a4c136:39915 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f03479, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:14:35,261 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:14:35,261 DEBUG [RS:2;7d6f26a4c136:32943 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e07382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:14:35,267 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T01:14:35,276 DEBUG [RS:1;7d6f26a4c136:40647 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7d6f26a4c136:40647 2024-11-27T01:14:35,275 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7d6f26a4c136,33239,1732670072670 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T01:14:35,280 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:14:35,281 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:14:35,281 DEBUG [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:14:35,285 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,33239,1732670072670 with port=40647, startcode=1732670073826 2024-11-27T01:14:35,291 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:14:35,291 DEBUG [RS:0;7d6f26a4c136:39915 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7d6f26a4c136:39915 2024-11-27T01:14:35,291 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:14:35,291 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:14:35,291 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:14:35,291 DEBUG [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:14:35,292 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:14:35,292 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:14:35,292 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7d6f26a4c136:0, corePoolSize=10, maxPoolSize=10 2024-11-27T01:14:35,292 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,293 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:14:35,293 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,294 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670105294 2024-11-27T01:14:35,295 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,33239,1732670072670 with port=39915, startcode=1732670073690 2024-11-27T01:14:35,296 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T01:14:35,297 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T01:14:35,299 DEBUG [RS:1;7d6f26a4c136:40647 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:14:35,300 DEBUG [RS:2;7d6f26a4c136:32943 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;7d6f26a4c136:32943 2024-11-27T01:14:35,300 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:14:35,300 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:14:35,300 DEBUG [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:14:35,301 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T01:14:35,301 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T01:14:35,302 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T01:14:35,302 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,33239,1732670072670 with port=32943, startcode=1732670073882 2024-11-27T01:14:35,302 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T01:14:35,302 DEBUG [RS:2;7d6f26a4c136:32943 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:14:35,303 DEBUG [RS:0;7d6f26a4c136:39915 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:14:35,305 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:35,305 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T01:14:35,304 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,307 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T01:14:35,309 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T01:14:35,309 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T01:14:35,315 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T01:14:35,315 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T01:14:35,322 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732670075321,5,FailOnTimeoutGroup] 2024-11-27T01:14:35,322 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:35,323 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T01:14:35,331 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732670075322,5,FailOnTimeoutGroup] 2024-11-27T01:14:35,331 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,331 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T01:14:35,333 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,333 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,359 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41567, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:14:35,359 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36693, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:14:35,360 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43415, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:14:35,370 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:35,374 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:35,398 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:35,398 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:35,402 DEBUG [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:35,403 DEBUG [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33923 2024-11-27T01:14:35,403 DEBUG [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:14:35,405 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:35,405 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:35,405 DEBUG [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:35,405 DEBUG [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33923 2024-11-27T01:14:35,406 DEBUG [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:14:35,410 DEBUG [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:35,410 DEBUG [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33923 2024-11-27T01:14:35,411 DEBUG [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:14:35,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:14:35,415 DEBUG [RS:0;7d6f26a4c136:39915 {}] zookeeper.ZKUtil(111): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:35,415 WARN [RS:0;7d6f26a4c136:39915 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:14:35,415 INFO [RS:0;7d6f26a4c136:39915 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:14:35,416 DEBUG [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:35,418 DEBUG [RS:1;7d6f26a4c136:40647 {}] zookeeper.ZKUtil(111): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:35,418 WARN [RS:1;7d6f26a4c136:40647 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:14:35,418 INFO [RS:1;7d6f26a4c136:40647 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:14:35,419 DEBUG [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:35,419 DEBUG [RS:2;7d6f26a4c136:32943 {}] zookeeper.ZKUtil(111): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:35,419 WARN [RS:2;7d6f26a4c136:32943 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:14:35,419 INFO [RS:2;7d6f26a4c136:32943 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:14:35,419 DEBUG [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:35,453 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,32943,1732670073882] 2024-11-27T01:14:35,453 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,39915,1732670073690] 2024-11-27T01:14:35,453 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,40647,1732670073826] 2024-11-27T01:14:35,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:14:35,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:14:35,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:14:35,465 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T01:14:35,466 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:35,507 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:14:35,508 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:14:35,523 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:14:35,533 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:14:35,538 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:14:35,539 INFO [RS:2;7d6f26a4c136:32943 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:14:35,540 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,540 INFO [RS:0;7d6f26a4c136:39915 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:14:35,540 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741832_1008 (size=32) 2024-11-27T01:14:35,543 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:14:35,543 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:14:35,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741832_1008 (size=32) 2024-11-27T01:14:35,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741832_1008 (size=32) 2024-11-27T01:14:35,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:35,551 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:14:35,552 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:14:35,552 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:14:35,552 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,553 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,553 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,553 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,553 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,553 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,553 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,553 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,554 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:14:35,554 INFO [RS:1;7d6f26a4c136:40647 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:14:35,554 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,554 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,554 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:14:35,554 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,554 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,553 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,555 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,555 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,555 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,555 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,555 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,555 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,555 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:14:35,555 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:14:35,555 DEBUG [RS:0;7d6f26a4c136:39915 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:14:35,555 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,556 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,556 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,556 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,556 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,556 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,556 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:14:35,557 DEBUG [RS:2;7d6f26a4c136:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:14:35,557 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:14:35,557 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,557 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,558 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,558 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,558 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,558 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,558 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:14:35,559 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,559 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,559 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,559 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,559 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,559 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:14:35,560 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:14:35,560 DEBUG [RS:1;7d6f26a4c136:40647 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:14:35,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T01:14:35,564 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T01:14:35,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:35,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:35,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T01:14:35,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T01:14:35,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:35,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:35,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T01:14:35,572 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,572 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,572 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,573 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,573 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,573 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,573 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,573 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,32943,1732670073882-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:14:35,573 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,573 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,573 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,573 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,40647,1732670073826-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:14:35,578 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T01:14:35,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:35,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:35,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T01:14:35,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T01:14:35,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:35,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:35,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T01:14:35,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740 2024-11-27T01:14:35,588 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,589 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,589 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,589 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,589 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740 2024-11-27T01:14:35,589 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,39915,1732670073690-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:14:35,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T01:14:35,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T01:14:35,594 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:14:35,597 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T01:14:35,615 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:35,617 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64795021, jitterRate=-0.03447894752025604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:14:35,619 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732670075548Initializing all the Stores at 1732670075551 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670075551Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670075557 (+6 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670075557Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670075557Cleaning up temporary data from old regions at 1732670075592 (+35 ms)Region opened successfully at 1732670075618 (+26 ms) 2024-11-27T01:14:35,619 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T01:14:35,619 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T01:14:35,619 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T01:14:35,619 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T01:14:35,619 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T01:14:35,620 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:14:35,622 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:14:35,623 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,40647,1732670073826-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,623 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,32943,1732670073882-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,624 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,624 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,624 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.Replication(171): 7d6f26a4c136,32943,1732670073882 started 2024-11-27T01:14:35,624 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.Replication(171): 7d6f26a4c136,40647,1732670073826 started 2024-11-27T01:14:35,625 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T01:14:35,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732670075619Disabling compacts and flushes for region at 1732670075619Disabling writes for close at 1732670075619Writing region close event to WAL at 1732670075625 (+6 ms)Closed at 1732670075625 2024-11-27T01:14:35,629 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:35,630 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T01:14:35,634 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:14:35,635 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,39915,1732670073690-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,635 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,635 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.Replication(171): 7d6f26a4c136,39915,1732670073690 started 2024-11-27T01:14:35,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T01:14:35,660 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:35,668 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T01:14:35,678 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,678 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,40647,1732670073826, RpcServer on 7d6f26a4c136/172.17.0.2:40647, sessionid=0x10156fd0ddf0002 2024-11-27T01:14:35,679 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,680 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,32943,1732670073882, RpcServer on 7d6f26a4c136/172.17.0.2:32943, sessionid=0x10156fd0ddf0003 2024-11-27T01:14:35,681 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:14:35,681 DEBUG [RS:2;7d6f26a4c136:32943 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:35,681 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,32943,1732670073882' 2024-11-27T01:14:35,681 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:14:35,681 DEBUG [RS:1;7d6f26a4c136:40647 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:35,682 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,40647,1732670073826' 2024-11-27T01:14:35,682 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:14:35,681 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:14:35,683 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:14:35,683 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:14:35,684 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:14:35,684 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:14:35,684 DEBUG [RS:1;7d6f26a4c136:40647 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:35,684 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,40647,1732670073826' 2024-11-27T01:14:35,684 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:14:35,684 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:14:35,684 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:14:35,684 DEBUG [RS:2;7d6f26a4c136:32943 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:35,684 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,32943,1732670073882' 2024-11-27T01:14:35,684 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:14:35,685 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:14:35,685 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:14:35,685 DEBUG [RS:1;7d6f26a4c136:40647 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:14:35,686 INFO [RS:1;7d6f26a4c136:40647 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:14:35,686 INFO [RS:1;7d6f26a4c136:40647 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:14:35,686 DEBUG [RS:2;7d6f26a4c136:32943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:14:35,686 INFO [RS:2;7d6f26a4c136:32943 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:14:35,686 INFO [RS:2;7d6f26a4c136:32943 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:14:35,695 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:35,695 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,39915,1732670073690, RpcServer on 7d6f26a4c136/172.17.0.2:39915, sessionid=0x10156fd0ddf0001 2024-11-27T01:14:35,695 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:14:35,695 DEBUG [RS:0;7d6f26a4c136:39915 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:35,695 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,39915,1732670073690' 2024-11-27T01:14:35,695 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:14:35,697 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:14:35,698 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:14:35,698 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:14:35,698 DEBUG [RS:0;7d6f26a4c136:39915 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:35,698 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,39915,1732670073690' 2024-11-27T01:14:35,698 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:14:35,699 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:14:35,700 DEBUG [RS:0;7d6f26a4c136:39915 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:14:35,701 INFO [RS:0;7d6f26a4c136:39915 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:14:35,701 INFO [RS:0;7d6f26a4c136:39915 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:14:35,793 INFO [RS:1;7d6f26a4c136:40647 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:14:35,793 INFO [RS:2;7d6f26a4c136:32943 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:14:35,797 INFO [RS:2;7d6f26a4c136:32943 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C32943%2C1732670073882, suffix=, logDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882, archiveDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs, maxLogs=32 2024-11-27T01:14:35,797 INFO [RS:1;7d6f26a4c136:40647 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C40647%2C1732670073826, suffix=, logDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,40647,1732670073826, archiveDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs, maxLogs=32 2024-11-27T01:14:35,802 INFO [RS:0;7d6f26a4c136:39915 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:14:35,805 INFO [RS:0;7d6f26a4c136:39915 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C39915%2C1732670073690, suffix=, logDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,39915,1732670073690, archiveDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs, maxLogs=32 2024-11-27T01:14:35,819 WARN [7d6f26a4c136:33239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T01:14:35,826 DEBUG [RS:2;7d6f26a4c136:32943 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882/7d6f26a4c136%2C32943%2C1732670073882.1732670075802, exclude list is [], retry=0 2024-11-27T01:14:35,826 DEBUG [RS:0;7d6f26a4c136:39915 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,39915,1732670073690/7d6f26a4c136%2C39915%2C1732670073690.1732670075807, exclude list is [], retry=0 2024-11-27T01:14:35,828 DEBUG [RS:1;7d6f26a4c136:40647 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,40647,1732670073826/7d6f26a4c136%2C40647%2C1732670073826.1732670075803, exclude list is [], retry=0 2024-11-27T01:14:35,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46391,DS-868d7a4e-f180-4a29-adfe-c9198fcd7493,DISK] 2024-11-27T01:14:35,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46391,DS-868d7a4e-f180-4a29-adfe-c9198fcd7493,DISK] 2024-11-27T01:14:35,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39429,DS-34aa0431-5d12-49f8-a64c-c90b8fe9f878,DISK] 2024-11-27T01:14:35,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40703,DS-ff782df8-ea5f-46cc-9d82-6b5a6f27a76c,DISK] 2024-11-27T01:14:35,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39429,DS-34aa0431-5d12-49f8-a64c-c90b8fe9f878,DISK] 2024-11-27T01:14:35,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40703,DS-ff782df8-ea5f-46cc-9d82-6b5a6f27a76c,DISK] 2024-11-27T01:14:35,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39429,DS-34aa0431-5d12-49f8-a64c-c90b8fe9f878,DISK] 2024-11-27T01:14:35,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46391,DS-868d7a4e-f180-4a29-adfe-c9198fcd7493,DISK] 2024-11-27T01:14:35,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40703,DS-ff782df8-ea5f-46cc-9d82-6b5a6f27a76c,DISK] 2024-11-27T01:14:35,869 INFO [RS:2;7d6f26a4c136:32943 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882/7d6f26a4c136%2C32943%2C1732670073882.1732670075802 2024-11-27T01:14:35,869 INFO [RS:0;7d6f26a4c136:39915 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,39915,1732670073690/7d6f26a4c136%2C39915%2C1732670073690.1732670075807 2024-11-27T01:14:35,873 DEBUG [RS:2;7d6f26a4c136:32943 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42521:42521),(127.0.0.1/127.0.0.1:34009:34009),(127.0.0.1/127.0.0.1:34837:34837)] 2024-11-27T01:14:35,873 DEBUG [RS:0;7d6f26a4c136:39915 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42521:42521),(127.0.0.1/127.0.0.1:34009:34009),(127.0.0.1/127.0.0.1:34837:34837)] 2024-11-27T01:14:35,876 INFO [RS:1;7d6f26a4c136:40647 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,40647,1732670073826/7d6f26a4c136%2C40647%2C1732670073826.1732670075803 2024-11-27T01:14:35,876 DEBUG [RS:1;7d6f26a4c136:40647 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42521:42521),(127.0.0.1/127.0.0.1:34009:34009),(127.0.0.1/127.0.0.1:34837:34837)] 2024-11-27T01:14:36,072 DEBUG [7d6f26a4c136:33239 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-27T01:14:36,083 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:14:36,090 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:14:36,090 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:14:36,090 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T01:14:36,090 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:14:36,090 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:14:36,090 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T01:14:36,090 INFO [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:14:36,090 INFO [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:14:36,090 INFO [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T01:14:36,090 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:14:36,099 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:36,108 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,32943,1732670073882, state=OPENING 2024-11-27T01:14:36,114 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T01:14:36,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:36,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:36,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:36,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:36,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:36,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:36,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:36,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:36,120 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:36,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,32943,1732670073882}] 2024-11-27T01:14:36,301 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:14:36,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48093, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:14:36,319 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T01:14:36,319 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:14:36,320 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T01:14:36,323 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C32943%2C1732670073882.meta, suffix=.meta, logDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882, archiveDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs, maxLogs=32 2024-11-27T01:14:36,347 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, exclude list is [], retry=0 2024-11-27T01:14:36,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39429,DS-34aa0431-5d12-49f8-a64c-c90b8fe9f878,DISK] 2024-11-27T01:14:36,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46391,DS-868d7a4e-f180-4a29-adfe-c9198fcd7493,DISK] 2024-11-27T01:14:36,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40703,DS-ff782df8-ea5f-46cc-9d82-6b5a6f27a76c,DISK] 2024-11-27T01:14:36,361 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta 2024-11-27T01:14:36,362 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34837:34837),(127.0.0.1/127.0.0.1:34009:34009),(127.0.0.1/127.0.0.1:42521:42521)] 2024-11-27T01:14:36,362 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:14:36,364 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T01:14:36,368 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T01:14:36,374 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T01:14:36,379 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T01:14:36,380 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:36,380 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T01:14:36,380 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T01:14:36,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T01:14:36,385 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T01:14:36,385 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:36,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:36,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T01:14:36,388 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T01:14:36,388 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:36,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:36,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T01:14:36,392 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T01:14:36,392 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:36,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:36,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T01:14:36,394 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T01:14:36,394 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:36,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:36,395 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T01:14:36,397 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740 2024-11-27T01:14:36,399 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740 2024-11-27T01:14:36,402 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T01:14:36,402 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T01:14:36,403 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:14:36,405 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T01:14:36,410 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63355889, jitterRate=-0.05592368543148041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:14:36,410 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T01:14:36,411 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732670076381Writing region info on filesystem at 1732670076381Initializing all the Stores at 1732670076383 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670076383Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670076384 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670076384Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670076384Cleaning up temporary data from old regions at 1732670076402 (+18 ms)Running coprocessor post-open hooks at 1732670076410 (+8 ms)Region opened successfully at 1732670076411 (+1 ms) 2024-11-27T01:14:36,420 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732670076292 2024-11-27T01:14:36,433 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T01:14:36,434 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T01:14:36,437 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:36,440 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,32943,1732670073882, state=OPEN 2024-11-27T01:14:36,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:36,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:36,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:36,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:36,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:36,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:36,444 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:36,444 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:36,444 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:36,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T01:14:36,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,32943,1732670073882 in 322 msec 2024-11-27T01:14:36,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T01:14:36,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 816 msec 2024-11-27T01:14:36,463 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:36,463 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T01:14:36,487 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:14:36,489 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1] 2024-11-27T01:14:36,520 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:14:36,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44725, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:14:36,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3710 sec 2024-11-27T01:14:36,556 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732670076556, completionTime=-1 2024-11-27T01:14:36,560 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-27T01:14:36,560 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:36,597 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-27T01:14:36,597 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670136597 2024-11-27T01:14:36,597 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670196597 2024-11-27T01:14:36,597 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 37 msec 2024-11-27T01:14:36,599 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:36,646 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,33239,1732670072670-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:36,646 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,33239,1732670072670-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:36,646 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,33239,1732670072670-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:36,648 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7d6f26a4c136:33239, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:36,651 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:36,655 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:36,658 DEBUG [master/7d6f26a4c136:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T01:14:36,702 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.702sec 2024-11-27T01:14:36,706 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T01:14:36,707 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T01:14:36,709 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T01:14:36,709 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T01:14:36,709 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T01:14:36,710 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,33239,1732670072670-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:14:36,711 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,33239,1732670072670-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T01:14:36,720 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T01:14:36,721 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T01:14:36,722 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,33239,1732670072670-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:14:36,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573cb019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:14:36,766 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T01:14:36,766 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T01:14:36,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,33239,-1 for getting cluster id 2024-11-27T01:14:36,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:14:36,790 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b70014f-cd86-42c2-939d-a21e25f574a1' 2024-11-27T01:14:36,795 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:14:36,796 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b70014f-cd86-42c2-939d-a21e25f574a1" 2024-11-27T01:14:36,796 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a0c39cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:14:36,796 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,33239,-1] 2024-11-27T01:14:36,800 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:14:36,803 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:36,811 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:14:36,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@451ec191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:14:36,816 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:14:36,828 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1] 2024-11-27T01:14:36,829 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:14:36,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:14:36,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:36,886 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:14:36,892 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:36,894 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32c7b5f8 2024-11-27T01:14:36,895 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:14:36,897 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:14:36,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:14:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:36,926 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T01:14:36,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-27T01:14:36,929 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:36,932 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T01:14:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:14:36,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741837_1013 (size=358) 2024-11-27T01:14:36,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741837_1013 (size=358) 2024-11-27T01:14:36,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741837_1013 (size=358) 2024-11-27T01:14:37,001 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3cfaabbc55e590863a7ecf45872977b3, NAME => 'test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,013 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bbecc4f063e3fd6151247c0507c5bef3, NAME => 'test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,016 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d17b306f903adc2b73f818b128d5cd67, NAME => 'test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:14:37,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741838_1014 (size=42) 2024-11-27T01:14:37,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741838_1014 (size=42) 2024-11-27T01:14:37,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741838_1014 (size=42) 2024-11-27T01:14:37,079 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,079 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3cfaabbc55e590863a7ecf45872977b3, disabling compactions & flushes 2024-11-27T01:14:37,079 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:37,079 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:37,079 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. after waiting 0 ms 2024-11-27T01:14:37,079 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:37,079 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:37,079 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3cfaabbc55e590863a7ecf45872977b3: Waiting for close lock at 1732670077079Disabling compacts and flushes for region at 1732670077079Disabling writes for close at 1732670077079Writing region close event to WAL at 1732670077079Closed at 1732670077079 2024-11-27T01:14:37,080 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 968f86830e42ee358d5e11672847740f, NAME => 'test,ddd,1732670076899.968f86830e42ee358d5e11672847740f.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741839_1015 (size=45) 2024-11-27T01:14:37,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741839_1015 (size=45) 2024-11-27T01:14:37,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741839_1015 (size=45) 2024-11-27T01:14:37,108 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d17b306f903adc2b73f818b128d5cd67, disabling compactions & flushes 2024-11-27T01:14:37,109 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:37,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:37,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. after waiting 0 ms 2024-11-27T01:14:37,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:37,109 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:37,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d17b306f903adc2b73f818b128d5cd67: Waiting for close lock at 1732670077109Disabling compacts and flushes for region at 1732670077109Disabling writes for close at 1732670077109Writing region close event to WAL at 1732670077109Closed at 1732670077109 2024-11-27T01:14:37,110 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 603677e75f480cfd74204fce51bd5359, NAME => 'test,eee,1732670076899.603677e75f480cfd74204fce51bd5359.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741840_1016 (size=45) 2024-11-27T01:14:37,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741840_1016 (size=45) 2024-11-27T01:14:37,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741840_1016 (size=45) 2024-11-27T01:14:37,132 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,132 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing bbecc4f063e3fd6151247c0507c5bef3, disabling compactions & flushes 2024-11-27T01:14:37,132 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:37,132 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:37,133 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. after waiting 0 ms 2024-11-27T01:14:37,133 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:37,133 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:37,133 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for bbecc4f063e3fd6151247c0507c5bef3: Waiting for close lock at 1732670077132Disabling compacts and flushes for region at 1732670077132Disabling writes for close at 1732670077133 (+1 ms)Writing region close event to WAL at 1732670077133Closed at 1732670077133 2024-11-27T01:14:37,134 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8e92cc2f48378cc4a64c15f36bcf6052, NAME => 'test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741842_1018 (size=45) 2024-11-27T01:14:37,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741842_1018 (size=45) 2024-11-27T01:14:37,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741841_1017 (size=45) 2024-11-27T01:14:37,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741841_1017 (size=45) 2024-11-27T01:14:37,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741841_1017 (size=45) 2024-11-27T01:14:37,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732670076899.968f86830e42ee358d5e11672847740f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 968f86830e42ee358d5e11672847740f, disabling compactions & flushes 2024-11-27T01:14:37,163 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:37,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:37,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. after waiting 0 ms 2024-11-27T01:14:37,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:37,163 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:37,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 968f86830e42ee358d5e11672847740f: Waiting for close lock at 1732670077163Disabling compacts and flushes for region at 1732670077163Disabling writes for close at 1732670077163Writing region close event to WAL at 1732670077163Closed at 1732670077163 2024-11-27T01:14:37,164 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 79f1ff709eae178b11e9bcfa1e4131b2, NAME => 'test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741842_1018 (size=45) 2024-11-27T01:14:37,166 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1732670076899.603677e75f480cfd74204fce51bd5359.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,167 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 603677e75f480cfd74204fce51bd5359, disabling compactions & flushes 2024-11-27T01:14:37,167 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:37,167 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:37,167 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. after waiting 0 ms 2024-11-27T01:14:37,167 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:37,167 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:37,167 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 603677e75f480cfd74204fce51bd5359: Waiting for close lock at 1732670077167Disabling compacts and flushes for region at 1732670077167Disabling writes for close at 1732670077167Writing region close event to WAL at 1732670077167Closed at 1732670077167 2024-11-27T01:14:37,168 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a05cacf8eaf88a4edb7255a00c86d78c, NAME => 'test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741843_1019 (size=45) 2024-11-27T01:14:37,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741843_1019 (size=45) 2024-11-27T01:14:37,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741843_1019 (size=45) 2024-11-27T01:14:37,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8e92cc2f48378cc4a64c15f36bcf6052, disabling compactions & flushes 2024-11-27T01:14:37,182 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:37,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:37,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. after waiting 0 ms 2024-11-27T01:14:37,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:37,182 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:37,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8e92cc2f48378cc4a64c15f36bcf6052: Waiting for close lock at 1732670077182Disabling compacts and flushes for region at 1732670077182Disabling writes for close at 1732670077182Writing region close event to WAL at 1732670077182Closed at 1732670077182 2024-11-27T01:14:37,183 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2fed3d0dc5a44acae9cbb7359cdf2051, NAME => 'test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741844_1020 (size=45) 2024-11-27T01:14:37,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741844_1020 (size=45) 2024-11-27T01:14:37,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741844_1020 (size=45) 2024-11-27T01:14:37,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741846_1022 (size=45) 2024-11-27T01:14:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741846_1022 (size=45) 2024-11-27T01:14:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741846_1022 (size=45) 2024-11-27T01:14:37,217 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,217 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 79f1ff709eae178b11e9bcfa1e4131b2, disabling compactions & flushes 2024-11-27T01:14:37,217 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:37,217 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:37,217 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. after waiting 0 ms 2024-11-27T01:14:37,217 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:37,217 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:37,218 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 79f1ff709eae178b11e9bcfa1e4131b2: Waiting for close lock at 1732670077217Disabling compacts and flushes for region at 1732670077217Disabling writes for close at 1732670077217Writing region close event to WAL at 1732670077217Closed at 1732670077217 2024-11-27T01:14:37,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741845_1021 (size=45) 2024-11-27T01:14:37,218 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eedb1cdc1fad3f780344bbfed739194f, NAME => 'test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,219 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,219 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2fed3d0dc5a44acae9cbb7359cdf2051, disabling compactions & flushes 2024-11-27T01:14:37,219 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:37,219 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:37,219 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. after waiting 0 ms 2024-11-27T01:14:37,219 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:37,219 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:37,219 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2fed3d0dc5a44acae9cbb7359cdf2051: Waiting for close lock at 1732670077219Disabling compacts and flushes for region at 1732670077219Disabling writes for close at 1732670077219Writing region close event to WAL at 1732670077219Closed at 1732670077219 2024-11-27T01:14:37,220 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 796f7fc8fad8d003c466a3f53b960815, NAME => 'test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741845_1021 (size=45) 2024-11-27T01:14:37,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741845_1021 (size=45) 2024-11-27T01:14:37,222 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,222 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a05cacf8eaf88a4edb7255a00c86d78c, disabling compactions & flushes 2024-11-27T01:14:37,222 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:37,222 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:37,223 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. after waiting 0 ms 2024-11-27T01:14:37,223 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:37,223 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:37,223 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a05cacf8eaf88a4edb7255a00c86d78c: Waiting for close lock at 1732670077222Disabling compacts and flushes for region at 1732670077222Disabling writes for close at 1732670077223 (+1 ms)Writing region close event to WAL at 1732670077223Closed at 1732670077223 2024-11-27T01:14:37,223 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8ae4c3f2c0fe2fac2f19395fca402eb5, NAME => 'test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:14:37,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741847_1023 (size=45) 2024-11-27T01:14:37,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741847_1023 (size=45) 2024-11-27T01:14:37,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741849_1025 (size=45) 2024-11-27T01:14:37,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741847_1023 (size=45) 2024-11-27T01:14:37,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741849_1025 (size=45) 2024-11-27T01:14:37,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741849_1025 (size=45) 2024-11-27T01:14:37,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741848_1024 (size=45) 2024-11-27T01:14:37,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741848_1024 (size=45) 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing eedb1cdc1fad3f780344bbfed739194f, disabling compactions & flushes 2024-11-27T01:14:37,294 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. after waiting 0 ms 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 796f7fc8fad8d003c466a3f53b960815, disabling compactions & flushes 2024-11-27T01:14:37,294 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:37,294 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. after waiting 0 ms 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for eedb1cdc1fad3f780344bbfed739194f: Waiting for close lock at 1732670077294Disabling compacts and flushes for region at 1732670077294Disabling writes for close at 1732670077294Writing region close event to WAL at 1732670077294Closed at 1732670077294 2024-11-27T01:14:37,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:37,294 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:37,295 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 796f7fc8fad8d003c466a3f53b960815: Waiting for close lock at 1732670077294Disabling compacts and flushes for region at 1732670077294Disabling writes for close at 1732670077294Writing region close event to WAL at 1732670077294Closed at 1732670077294 2024-11-27T01:14:37,295 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4a61d50f100ead183465a200984a5e77, NAME => 'test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,296 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8af3d7357dfa0f8aa36ba6b01663d6f6, NAME => 'test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741848_1024 (size=45) 2024-11-27T01:14:37,296 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,296 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8ae4c3f2c0fe2fac2f19395fca402eb5, disabling compactions & flushes 2024-11-27T01:14:37,296 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:37,296 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:37,296 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. after waiting 0 ms 2024-11-27T01:14:37,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:37,297 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:37,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8ae4c3f2c0fe2fac2f19395fca402eb5: Waiting for close lock at 1732670077296Disabling compacts and flushes for region at 1732670077296Disabling writes for close at 1732670077297 (+1 ms)Writing region close event to WAL at 1732670077297Closed at 1732670077297 2024-11-27T01:14:37,298 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b272925d8b30354bd56e190c0c041139, NAME => 'test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741850_1026 (size=45) 2024-11-27T01:14:37,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4a61d50f100ead183465a200984a5e77, disabling compactions & flushes 2024-11-27T01:14:37,344 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:37,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:37,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. after waiting 0 ms 2024-11-27T01:14:37,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:37,344 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:37,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4a61d50f100ead183465a200984a5e77: Waiting for close lock at 1732670077344Disabling compacts and flushes for region at 1732670077344Disabling writes for close at 1732670077344Writing region close event to WAL at 1732670077344Closed at 1732670077344 2024-11-27T01:14:37,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741850_1026 (size=45) 2024-11-27T01:14:37,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741850_1026 (size=45) 2024-11-27T01:14:37,345 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8695d5f6c29ed614cc7e97c2ba4c6724, NAME => 'test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741852_1028 (size=45) 2024-11-27T01:14:37,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741852_1028 (size=45) 2024-11-27T01:14:37,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741852_1028 (size=45) 2024-11-27T01:14:37,351 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,351 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b272925d8b30354bd56e190c0c041139, disabling compactions & flushes 2024-11-27T01:14:37,351 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:37,351 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:37,351 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. after waiting 0 ms 2024-11-27T01:14:37,351 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:37,351 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:37,351 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b272925d8b30354bd56e190c0c041139: Waiting for close lock at 1732670077351Disabling compacts and flushes for region at 1732670077351Disabling writes for close at 1732670077351Writing region close event to WAL at 1732670077351Closed at 1732670077351 2024-11-27T01:14:37,352 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => edc306f98fb391a3fdc7623063a1cbf8, NAME => 'test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741851_1027 (size=45) 2024-11-27T01:14:37,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741851_1027 (size=45) 2024-11-27T01:14:37,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741851_1027 (size=45) 2024-11-27T01:14:37,370 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,370 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8af3d7357dfa0f8aa36ba6b01663d6f6, disabling compactions & flushes 2024-11-27T01:14:37,370 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:37,370 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:37,371 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. after waiting 0 ms 2024-11-27T01:14:37,371 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:37,371 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:37,371 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8af3d7357dfa0f8aa36ba6b01663d6f6: Waiting for close lock at 1732670077370Disabling compacts and flushes for region at 1732670077370Disabling writes for close at 1732670077371 (+1 ms)Writing region close event to WAL at 1732670077371Closed at 1732670077371 2024-11-27T01:14:37,371 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4b3cb4626ebbddc356f324f4faf22f80, NAME => 'test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741854_1030 (size=45) 2024-11-27T01:14:37,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741854_1030 (size=45) 2024-11-27T01:14:37,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741854_1030 (size=45) 2024-11-27T01:14:37,409 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing edc306f98fb391a3fdc7623063a1cbf8, disabling compactions & flushes 2024-11-27T01:14:37,410 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:37,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:37,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. after waiting 0 ms 2024-11-27T01:14:37,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:37,410 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:37,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for edc306f98fb391a3fdc7623063a1cbf8: Waiting for close lock at 1732670077409Disabling compacts and flushes for region at 1732670077409Disabling writes for close at 1732670077410 (+1 ms)Writing region close event to WAL at 1732670077410Closed at 1732670077410 2024-11-27T01:14:37,411 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b85ee4f1fa33a9cbdcb45c32b6176c88, NAME => 'test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741853_1029 (size=45) 2024-11-27T01:14:37,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741853_1029 (size=45) 2024-11-27T01:14:37,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741853_1029 (size=45) 2024-11-27T01:14:37,426 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,426 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8695d5f6c29ed614cc7e97c2ba4c6724, disabling compactions & flushes 2024-11-27T01:14:37,426 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:37,426 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:37,426 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. after waiting 0 ms 2024-11-27T01:14:37,426 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:37,426 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:37,426 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8695d5f6c29ed614cc7e97c2ba4c6724: Waiting for close lock at 1732670077426Disabling compacts and flushes for region at 1732670077426Disabling writes for close at 1732670077426Writing region close event to WAL at 1732670077426Closed at 1732670077426 2024-11-27T01:14:37,427 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1229242f433b46943919d6643a95d4a4, NAME => 'test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741855_1031 (size=45) 2024-11-27T01:14:37,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741855_1031 (size=45) 2024-11-27T01:14:37,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741855_1031 (size=45) 2024-11-27T01:14:37,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4b3cb4626ebbddc356f324f4faf22f80, disabling compactions & flushes 2024-11-27T01:14:37,446 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:37,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:37,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. after waiting 0 ms 2024-11-27T01:14:37,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:37,446 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:37,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4b3cb4626ebbddc356f324f4faf22f80: Waiting for close lock at 1732670077446Disabling compacts and flushes for region at 1732670077446Disabling writes for close at 1732670077446Writing region close event to WAL at 1732670077446Closed at 1732670077446 2024-11-27T01:14:37,447 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0caf01005a5fb704a147ee46c7227b3e, NAME => 'test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741856_1032 (size=45) 2024-11-27T01:14:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741856_1032 (size=45) 2024-11-27T01:14:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741856_1032 (size=45) 2024-11-27T01:14:37,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b85ee4f1fa33a9cbdcb45c32b6176c88, disabling compactions & flushes 2024-11-27T01:14:37,458 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:37,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:37,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. after waiting 0 ms 2024-11-27T01:14:37,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:37,458 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:37,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b85ee4f1fa33a9cbdcb45c32b6176c88: Waiting for close lock at 1732670077458Disabling compacts and flushes for region at 1732670077458Disabling writes for close at 1732670077458Writing region close event to WAL at 1732670077458Closed at 1732670077458 2024-11-27T01:14:37,459 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fc23d7fa863cb970ac753db9e4e26b26, NAME => 'test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741857_1033 (size=45) 2024-11-27T01:14:37,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741857_1033 (size=45) 2024-11-27T01:14:37,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741857_1033 (size=45) 2024-11-27T01:14:37,479 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,479 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1229242f433b46943919d6643a95d4a4, disabling compactions & flushes 2024-11-27T01:14:37,479 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:37,479 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:37,479 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. after waiting 0 ms 2024-11-27T01:14:37,479 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:37,480 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:37,480 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1229242f433b46943919d6643a95d4a4: Waiting for close lock at 1732670077479Disabling compacts and flushes for region at 1732670077479Disabling writes for close at 1732670077479Writing region close event to WAL at 1732670077479Closed at 1732670077479 2024-11-27T01:14:37,480 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ff2b8d760c6f5acc07476412531886a0, NAME => 'test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741858_1034 (size=45) 2024-11-27T01:14:37,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741858_1034 (size=45) 2024-11-27T01:14:37,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741858_1034 (size=45) 2024-11-27T01:14:37,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0caf01005a5fb704a147ee46c7227b3e, disabling compactions & flushes 2024-11-27T01:14:37,497 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:37,498 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:37,498 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. after waiting 0 ms 2024-11-27T01:14:37,498 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:37,498 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:37,498 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0caf01005a5fb704a147ee46c7227b3e: Waiting for close lock at 1732670077497Disabling compacts and flushes for region at 1732670077497Disabling writes for close at 1732670077498 (+1 ms)Writing region close event to WAL at 1732670077498Closed at 1732670077498 2024-11-27T01:14:37,499 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 335860f4847882dab27e1db49323c60b, NAME => 'test,xxx,1732670076899.335860f4847882dab27e1db49323c60b.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741859_1035 (size=45) 2024-11-27T01:14:37,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741859_1035 (size=45) 2024-11-27T01:14:37,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741859_1035 (size=45) 2024-11-27T01:14:37,512 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,512 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fc23d7fa863cb970ac753db9e4e26b26, disabling compactions & flushes 2024-11-27T01:14:37,512 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:37,512 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:37,512 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. after waiting 0 ms 2024-11-27T01:14:37,512 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:37,512 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:37,512 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fc23d7fa863cb970ac753db9e4e26b26: Waiting for close lock at 1732670077512Disabling compacts and flushes for region at 1732670077512Disabling writes for close at 1732670077512Writing region close event to WAL at 1732670077512Closed at 1732670077512 2024-11-27T01:14:37,513 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => adbccc292c5b7e61d47646b64c419a44, NAME => 'test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741860_1036 (size=45) 2024-11-27T01:14:37,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741860_1036 (size=45) 2024-11-27T01:14:37,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741860_1036 (size=45) 2024-11-27T01:14:37,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ff2b8d760c6f5acc07476412531886a0, disabling compactions & flushes 2024-11-27T01:14:37,527 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:37,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:37,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. after waiting 0 ms 2024-11-27T01:14:37,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:37,527 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:37,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ff2b8d760c6f5acc07476412531886a0: Waiting for close lock at 1732670077527Disabling compacts and flushes for region at 1732670077527Disabling writes for close at 1732670077527Writing region close event to WAL at 1732670077527Closed at 1732670077527 2024-11-27T01:14:37,528 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 360d0b423433ec91f787be3bde665606, NAME => 'test,zzz,1732670076899.360d0b423433ec91f787be3bde665606.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d 2024-11-27T01:14:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:14:37,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741861_1037 (size=45) 2024-11-27T01:14:37,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741862_1038 (size=45) 2024-11-27T01:14:37,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741861_1037 (size=45) 2024-11-27T01:14:37,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741862_1038 (size=45) 2024-11-27T01:14:37,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741861_1037 (size=45) 2024-11-27T01:14:37,574 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1732670076899.335860f4847882dab27e1db49323c60b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,575 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 335860f4847882dab27e1db49323c60b, disabling compactions & flushes 2024-11-27T01:14:37,575 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:37,575 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:37,575 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. after waiting 0 ms 2024-11-27T01:14:37,575 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:37,575 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:37,575 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 335860f4847882dab27e1db49323c60b: Waiting for close lock at 1732670077575Disabling compacts and flushes for region at 1732670077575Disabling writes for close at 1732670077575Writing region close event to WAL at 1732670077575Closed at 1732670077575 2024-11-27T01:14:37,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing adbccc292c5b7e61d47646b64c419a44, disabling compactions & flushes 2024-11-27T01:14:37,577 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:37,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:37,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. after waiting 0 ms 2024-11-27T01:14:37,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:37,577 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:37,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for adbccc292c5b7e61d47646b64c419a44: Waiting for close lock at 1732670077577Disabling compacts and flushes for region at 1732670077577Disabling writes for close at 1732670077577Writing region close event to WAL at 1732670077577Closed at 1732670077577 2024-11-27T01:14:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741862_1038 (size=45) 2024-11-27T01:14:37,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741863_1039 (size=42) 2024-11-27T01:14:37,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741863_1039 (size=42) 2024-11-27T01:14:37,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741863_1039 (size=42) 2024-11-27T01:14:37,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1732670076899.360d0b423433ec91f787be3bde665606.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:37,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 360d0b423433ec91f787be3bde665606, disabling compactions & flushes 2024-11-27T01:14:37,603 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:37,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:37,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. after waiting 0 ms 2024-11-27T01:14:37,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:37,603 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:37,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 360d0b423433ec91f787be3bde665606: Waiting for close lock at 1732670077603Disabling compacts and flushes for region at 1732670077603Disabling writes for close at 1732670077603Writing region close event to WAL at 1732670077603Closed at 1732670077603 2024-11-27T01:14:37,608 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T01:14:37,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732670076899.968f86830e42ee358d5e11672847740f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732670076899.603677e75f480cfd74204fce51bd5359.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,617 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,617 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,617 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,617 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,618 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,619 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,619 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,619 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,619 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,620 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,620 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,621 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,623 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732670076899.335860f4847882dab27e1db49323c60b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,623 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,623 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732670076899.360d0b423433ec91f787be3bde665606.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732670077608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670077608"}]},"ts":"1732670077608"} 2024-11-27T01:14:37,700 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-27T01:14:37,705 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T01:14:37,707 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732670077705"}]},"ts":"1732670077705"} 2024-11-27T01:14:37,717 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-27T01:14:37,718 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:14:37,744 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:14:37,744 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:14:37,744 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T01:14:37,744 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:14:37,744 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:14:37,744 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T01:14:37,744 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:14:37,744 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:14:37,744 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T01:14:37,744 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:14:37,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3cfaabbc55e590863a7ecf45872977b3, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d17b306f903adc2b73f818b128d5cd67, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=968f86830e42ee358d5e11672847740f, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=603677e75f480cfd74204fce51bd5359, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79f1ff709eae178b11e9bcfa1e4131b2, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2fed3d0dc5a44acae9cbb7359cdf2051, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eedb1cdc1fad3f780344bbfed739194f, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=796f7fc8fad8d003c466a3f53b960815, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a61d50f100ead183465a200984a5e77, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b272925d8b30354bd56e190c0c041139, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8695d5f6c29ed614cc7e97c2ba4c6724, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edc306f98fb391a3fdc7623063a1cbf8, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b85ee4f1fa33a9cbdcb45c32b6176c88, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1229242f433b46943919d6643a95d4a4, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc23d7fa863cb970ac753db9e4e26b26, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff2b8d760c6f5acc07476412531886a0, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=adbccc292c5b7e61d47646b64c419a44, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=360d0b423433ec91f787be3bde665606, ASSIGN}] 2024-11-27T01:14:37,761 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=360d0b423433ec91f787be3bde665606, ASSIGN 2024-11-27T01:14:37,763 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=360d0b423433ec91f787be3bde665606, ASSIGN; state=OFFLINE, location=7d6f26a4c136,40647,1732670073826; forceNewPlan=false, retain=false 2024-11-27T01:14:37,768 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=adbccc292c5b7e61d47646b64c419a44, ASSIGN 2024-11-27T01:14:37,774 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=adbccc292c5b7e61d47646b64c419a44, ASSIGN; state=OFFLINE, location=7d6f26a4c136,39915,1732670073690; forceNewPlan=false, retain=false 2024-11-27T01:14:37,777 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:37,779 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN; state=OFFLINE, location=7d6f26a4c136,32943,1732670073882; forceNewPlan=false, retain=false 2024-11-27T01:14:37,782 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff2b8d760c6f5acc07476412531886a0, ASSIGN 2024-11-27T01:14:37,783 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ff2b8d760c6f5acc07476412531886a0, ASSIGN; state=OFFLINE, location=7d6f26a4c136,40647,1732670073826; forceNewPlan=false, retain=false 2024-11-27T01:14:37,785 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc23d7fa863cb970ac753db9e4e26b26, ASSIGN 2024-11-27T01:14:37,787 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fc23d7fa863cb970ac753db9e4e26b26, ASSIGN; state=OFFLINE, location=7d6f26a4c136,39915,1732670073690; forceNewPlan=false, retain=false 2024-11-27T01:14:37,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:37,798 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN; state=OFFLINE, location=7d6f26a4c136,32943,1732670073882; forceNewPlan=false, retain=false 2024-11-27T01:14:37,804 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1229242f433b46943919d6643a95d4a4, ASSIGN 2024-11-27T01:14:37,806 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1229242f433b46943919d6643a95d4a4, ASSIGN; state=OFFLINE, location=7d6f26a4c136,40647,1732670073826; forceNewPlan=false, retain=false 2024-11-27T01:14:37,812 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b85ee4f1fa33a9cbdcb45c32b6176c88, ASSIGN 2024-11-27T01:14:37,815 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b85ee4f1fa33a9cbdcb45c32b6176c88, ASSIGN; state=OFFLINE, location=7d6f26a4c136,39915,1732670073690; forceNewPlan=false, retain=false 2024-11-27T01:14:37,819 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:37,823 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN; state=OFFLINE, location=7d6f26a4c136,32943,1732670073882; forceNewPlan=false, retain=false 2024-11-27T01:14:37,825 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edc306f98fb391a3fdc7623063a1cbf8, ASSIGN 2024-11-27T01:14:37,827 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=edc306f98fb391a3fdc7623063a1cbf8, ASSIGN; state=OFFLINE, location=7d6f26a4c136,39915,1732670073690; forceNewPlan=false, retain=false 2024-11-27T01:14:37,829 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8695d5f6c29ed614cc7e97c2ba4c6724, ASSIGN 2024-11-27T01:14:37,832 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8695d5f6c29ed614cc7e97c2ba4c6724, ASSIGN; state=OFFLINE, location=7d6f26a4c136,40647,1732670073826; forceNewPlan=false, retain=false 2024-11-27T01:14:37,834 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b272925d8b30354bd56e190c0c041139, ASSIGN 2024-11-27T01:14:37,840 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b272925d8b30354bd56e190c0c041139, ASSIGN; state=OFFLINE, location=7d6f26a4c136,40647,1732670073826; forceNewPlan=false, retain=false 2024-11-27T01:14:37,843 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:37,846 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN; state=OFFLINE, location=7d6f26a4c136,32943,1732670073882; forceNewPlan=false, retain=false 2024-11-27T01:14:37,848 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a61d50f100ead183465a200984a5e77, ASSIGN 2024-11-27T01:14:37,853 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4a61d50f100ead183465a200984a5e77, ASSIGN; state=OFFLINE, location=7d6f26a4c136,39915,1732670073690; forceNewPlan=false, retain=false 2024-11-27T01:14:37,860 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:37,863 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN; state=OFFLINE, location=7d6f26a4c136,32943,1732670073882; forceNewPlan=false, retain=false 2024-11-27T01:14:37,865 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=796f7fc8fad8d003c466a3f53b960815, ASSIGN 2024-11-27T01:14:37,867 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=796f7fc8fad8d003c466a3f53b960815, ASSIGN; state=OFFLINE, location=7d6f26a4c136,40647,1732670073826; forceNewPlan=false, retain=false 2024-11-27T01:14:37,871 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eedb1cdc1fad3f780344bbfed739194f, ASSIGN 2024-11-27T01:14:37,879 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eedb1cdc1fad3f780344bbfed739194f, ASSIGN; state=OFFLINE, location=7d6f26a4c136,39915,1732670073690; forceNewPlan=false, retain=false 2024-11-27T01:14:37,886 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2fed3d0dc5a44acae9cbb7359cdf2051, ASSIGN 2024-11-27T01:14:37,888 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2fed3d0dc5a44acae9cbb7359cdf2051, ASSIGN; state=OFFLINE, location=7d6f26a4c136,40647,1732670073826; forceNewPlan=false, retain=false 2024-11-27T01:14:37,891 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:37,894 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN; state=OFFLINE, location=7d6f26a4c136,32943,1732670073882; forceNewPlan=false, retain=false 2024-11-27T01:14:37,899 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79f1ff709eae178b11e9bcfa1e4131b2, ASSIGN 2024-11-27T01:14:37,904 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=79f1ff709eae178b11e9bcfa1e4131b2, ASSIGN; state=OFFLINE, location=7d6f26a4c136,39915,1732670073690; forceNewPlan=false, retain=false 2024-11-27T01:14:37,907 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:37,909 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN; state=OFFLINE, location=7d6f26a4c136,32943,1732670073882; forceNewPlan=false, retain=false 2024-11-27T01:14:37,911 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=603677e75f480cfd74204fce51bd5359, ASSIGN 2024-11-27T01:14:37,915 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=603677e75f480cfd74204fce51bd5359, ASSIGN; state=OFFLINE, location=7d6f26a4c136,40647,1732670073826; forceNewPlan=false, retain=false 2024-11-27T01:14:37,917 INFO [7d6f26a4c136:33239 {}] balancer.BaseLoadBalancer(388): Reassigned 21 regions. 21 retained the pre-restart assignment. 2024-11-27T01:14:37,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=360d0b423433ec91f787be3bde665606, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:37,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ff2b8d760c6f5acc07476412531886a0, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:37,924 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1229242f433b46943919d6643a95d4a4, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:37,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:37,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=b272925d8b30354bd56e190c0c041139, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:37,926 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=796f7fc8fad8d003c466a3f53b960815, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:37,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:37,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=adbccc292c5b7e61d47646b64c419a44, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:37,928 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:37,931 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:37,932 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:37,932 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=4a61d50f100ead183465a200984a5e77, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:37,933 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=eedb1cdc1fad3f780344bbfed739194f, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:37,935 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:37,936 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=335860f4847882dab27e1db49323c60b, regionState=OPENING, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:37,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=360d0b423433ec91f787be3bde665606, ASSIGN because future has completed 2024-11-27T01:14:37,942 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:37,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 360d0b423433ec91f787be3bde665606, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:37,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ff2b8d760c6f5acc07476412531886a0, ASSIGN because future has completed 2024-11-27T01:14:37,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1229242f433b46943919d6643a95d4a4, ASSIGN because future has completed 2024-11-27T01:14:37,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8695d5f6c29ed614cc7e97c2ba4c6724, ASSIGN because future has completed 2024-11-27T01:14:37,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8695d5f6c29ed614cc7e97c2ba4c6724, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:37,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b272925d8b30354bd56e190c0c041139, ASSIGN because future has completed 2024-11-27T01:14:37,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=796f7fc8fad8d003c466a3f53b960815, ASSIGN because future has completed 2024-11-27T01:14:37,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2fed3d0dc5a44acae9cbb7359cdf2051, ASSIGN because future has completed 2024-11-27T01:14:37,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=adbccc292c5b7e61d47646b64c419a44, ASSIGN because future has completed 2024-11-27T01:14:37,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbccc292c5b7e61d47646b64c419a44, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:37,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fc23d7fa863cb970ac753db9e4e26b26, ASSIGN because future has completed 2024-11-27T01:14:37,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b85ee4f1fa33a9cbdcb45c32b6176c88, ASSIGN because future has completed 2024-11-27T01:14:37,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=edc306f98fb391a3fdc7623063a1cbf8, ASSIGN because future has completed 2024-11-27T01:14:38,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4a61d50f100ead183465a200984a5e77, ASSIGN because future has completed 2024-11-27T01:14:38,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a61d50f100ead183465a200984a5e77, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:38,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eedb1cdc1fad3f780344bbfed739194f, ASSIGN because future has completed 2024-11-27T01:14:38,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=79f1ff709eae178b11e9bcfa1e4131b2, ASSIGN because future has completed 2024-11-27T01:14:38,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN because future has completed 2024-11-27T01:14:38,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,32943,1732670073882}] 2024-11-27T01:14:38,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN because future has completed 2024-11-27T01:14:38,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,32943,1732670073882}] 2024-11-27T01:14:38,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79f1ff709eae178b11e9bcfa1e4131b2, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:38,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure eedb1cdc1fad3f780344bbfed739194f, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:38,068 INFO [7d6f26a4c136:33239 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T01:14:38,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=603677e75f480cfd74204fce51bd5359, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc306f98fb391a3fdc7623063a1cbf8, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:14:38,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=603677e75f480cfd74204fce51bd5359, ASSIGN because future has completed 2024-11-27T01:14:38,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 603677e75f480cfd74204fce51bd5359, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:38,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure b85ee4f1fa33a9cbdcb45c32b6176c88, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:38,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc23d7fa863cb970ac753db9e4e26b26, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:38,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fed3d0dc5a44acae9cbb7359cdf2051, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:38,121 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:14:38,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 796f7fc8fad8d003c466a3f53b960815, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:38,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure b272925d8b30354bd56e190c0c041139, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:38,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42659, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:14:38,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1229242f433b46943919d6643a95d4a4, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:38,153 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:38,153 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 360d0b423433ec91f787be3bde665606, NAME => 'test,zzz,1732670076899.360d0b423433ec91f787be3bde665606.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-27T01:14:38,153 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,154 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1732670076899.360d0b423433ec91f787be3bde665606.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,154 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,154 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff2b8d760c6f5acc07476412531886a0, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:38,165 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,165 INFO [StoreOpener-360d0b423433ec91f787be3bde665606-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,165 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:14:38,166 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,167 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,168 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,169 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,169 INFO [StoreOpener-360d0b423433ec91f787be3bde665606-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 360d0b423433ec91f787be3bde665606 columnFamilyName family 2024-11-27T01:14:38,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN because future has completed 2024-11-27T01:14:38,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,32943,1732670073882}] 2024-11-27T01:14:38,171 DEBUG [StoreOpener-360d0b423433ec91f787be3bde665606-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,173 INFO [StoreOpener-360d0b423433ec91f787be3bde665606-1 {}] regionserver.HStore(327): Store=360d0b423433ec91f787be3bde665606/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,173 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN because future has completed 2024-11-27T01:14:38,175 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38889, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:14:38,176 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,177 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,177 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN because future has completed 2024-11-27T01:14:38,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN because future has completed 2024-11-27T01:14:38,183 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN because future has completed 2024-11-27T01:14:38,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e92cc2f48378cc4a64c15f36bcf6052, server=7d6f26a4c136,32943,1732670073882}] 2024-11-27T01:14:38,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,32943,1732670073882}] 2024-11-27T01:14:38,208 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:38,208 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => adbccc292c5b7e61d47646b64c419a44, NAME => 'test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-27T01:14:38,209 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,209 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,210 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,210 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,218 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:38,219 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 0caf01005a5fb704a147ee46c7227b3e, NAME => 'test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-27T01:14:38,220 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,220 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,220 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,220 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,32943,1732670073882}] 2024-11-27T01:14:38,224 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/360d0b423433ec91f787be3bde665606/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,225 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 360d0b423433ec91f787be3bde665606; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58933600, jitterRate=-0.12182092666625977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,226 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:38,227 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 360d0b423433ec91f787be3bde665606: Running coprocessor pre-open hook at 1732670078154Writing region info on filesystem at 1732670078154Initializing all the Stores at 1732670078156 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078156Cleaning up temporary data from old regions at 1732670078177 (+21 ms)Running coprocessor post-open hooks at 1732670078226 (+49 ms)Region opened successfully at 1732670078227 (+1 ms) 2024-11-27T01:14:38,230 INFO [StoreOpener-adbccc292c5b7e61d47646b64c419a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,230 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732670076899.360d0b423433ec91f787be3bde665606., pid=31, masterSystemTime=1732670078121 2024-11-27T01:14:38,237 INFO [StoreOpener-adbccc292c5b7e61d47646b64c419a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbccc292c5b7e61d47646b64c419a44 columnFamilyName family 2024-11-27T01:14:38,237 DEBUG [StoreOpener-adbccc292c5b7e61d47646b64c419a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,238 INFO [StoreOpener-adbccc292c5b7e61d47646b64c419a44-1 {}] regionserver.HStore(327): Store=adbccc292c5b7e61d47646b64c419a44/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,238 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,240 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,240 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:38,240 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:38,241 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,241 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:38,241 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 8695d5f6c29ed614cc7e97c2ba4c6724, NAME => 'test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-27T01:14:38,241 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,241 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,241 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,242 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,242 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,242 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,244 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=360d0b423433ec91f787be3bde665606, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,32943,1732670073882}] 2024-11-27T01:14:38,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 360d0b423433ec91f787be3bde665606, server=7d6f26a4c136,40647,1732670073826 because future has completed 2024-11-27T01:14:38,254 INFO [StoreOpener-8695d5f6c29ed614cc7e97c2ba4c6724-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,257 INFO [StoreOpener-8695d5f6c29ed614cc7e97c2ba4c6724-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8695d5f6c29ed614cc7e97c2ba4c6724 columnFamilyName family 2024-11-27T01:14:38,257 DEBUG [StoreOpener-8695d5f6c29ed614cc7e97c2ba4c6724-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,258 INFO [StoreOpener-8695d5f6c29ed614cc7e97c2ba4c6724-1 {}] regionserver.HStore(327): Store=8695d5f6c29ed614cc7e97c2ba4c6724/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,258 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,259 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,260 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,261 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,261 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,264 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,265 INFO [StoreOpener-0caf01005a5fb704a147ee46c7227b3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-27T01:14:38,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 360d0b423433ec91f787be3bde665606, server=7d6f26a4c136,40647,1732670073826 in 323 msec 2024-11-27T01:14:38,274 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/adbccc292c5b7e61d47646b64c419a44/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,275 INFO [StoreOpener-0caf01005a5fb704a147ee46c7227b3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0caf01005a5fb704a147ee46c7227b3e columnFamilyName family 2024-11-27T01:14:38,275 DEBUG [StoreOpener-0caf01005a5fb704a147ee46c7227b3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,275 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened adbccc292c5b7e61d47646b64c419a44; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61482949, jitterRate=-0.08383266627788544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,276 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:38,276 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for adbccc292c5b7e61d47646b64c419a44: Running coprocessor pre-open hook at 1732670078211Writing region info on filesystem at 1732670078211Initializing all the Stores at 1732670078213 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078214 (+1 ms)Cleaning up temporary data from old regions at 1732670078241 (+27 ms)Running coprocessor post-open hooks at 1732670078276 (+35 ms)Region opened successfully at 1732670078276 2024-11-27T01:14:38,277 INFO [StoreOpener-0caf01005a5fb704a147ee46c7227b3e-1 {}] regionserver.HStore(327): Store=0caf01005a5fb704a147ee46c7227b3e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,277 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,279 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,279 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44., pid=33, masterSystemTime=1732670078164 2024-11-27T01:14:38,279 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,280 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8695d5f6c29ed614cc7e97c2ba4c6724/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=360d0b423433ec91f787be3bde665606, ASSIGN in 525 msec 2024-11-27T01:14:38,280 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,280 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,280 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=968f86830e42ee358d5e11672847740f, ASSIGN 2024-11-27T01:14:38,281 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 8695d5f6c29ed614cc7e97c2ba4c6724; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69961178, jitterRate=0.042502790689468384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,281 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:38,281 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 8695d5f6c29ed614cc7e97c2ba4c6724: Running coprocessor pre-open hook at 1732670078242Writing region info on filesystem at 1732670078242Initializing all the Stores at 1732670078244 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078244Cleaning up temporary data from old regions at 1732670078261 (+17 ms)Running coprocessor post-open hooks at 1732670078281 (+20 ms)Region opened successfully at 1732670078281 2024-11-27T01:14:38,283 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724., pid=32, masterSystemTime=1732670078121 2024-11-27T01:14:38,284 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=968f86830e42ee358d5e11672847740f, ASSIGN; state=OFFLINE, location=7d6f26a4c136,39915,1732670073690; forceNewPlan=false, retain=false 2024-11-27T01:14:38,286 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,288 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:38,288 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:38,288 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:38,288 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a61d50f100ead183465a200984a5e77, NAME => 'test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-27T01:14:38,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:38,289 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,289 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:38,289 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,289 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:38,289 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,289 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,289 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:38,289 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 603677e75f480cfd74204fce51bd5359, NAME => 'test,eee,1732670076899.603677e75f480cfd74204fce51bd5359.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-27T01:14:38,290 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,290 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,eee,1732670076899.603677e75f480cfd74204fce51bd5359.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,290 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,290 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbccc292c5b7e61d47646b64c419a44, server=7d6f26a4c136,39915,1732670073690 because future has completed 2024-11-27T01:14:38,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8695d5f6c29ed614cc7e97c2ba4c6724, server=7d6f26a4c136,40647,1732670073826 because future has completed 2024-11-27T01:14:38,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=29 2024-11-27T01:14:38,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure adbccc292c5b7e61d47646b64c419a44, server=7d6f26a4c136,39915,1732670073690 in 310 msec 2024-11-27T01:14:38,304 INFO [StoreOpener-4a61d50f100ead183465a200984a5e77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=adbccc292c5b7e61d47646b64c419a44, ASSIGN in 554 msec 2024-11-27T01:14:38,307 INFO [StoreOpener-4a61d50f100ead183465a200984a5e77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a61d50f100ead183465a200984a5e77 columnFamilyName family 2024-11-27T01:14:38,308 DEBUG [StoreOpener-4a61d50f100ead183465a200984a5e77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,308 INFO [StoreOpener-4a61d50f100ead183465a200984a5e77-1 {}] regionserver.HStore(327): Store=4a61d50f100ead183465a200984a5e77/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,309 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,310 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,311 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,312 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,312 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,315 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-11-27T01:14:38,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 8695d5f6c29ed614cc7e97c2ba4c6724, server=7d6f26a4c136,40647,1732670073826 in 339 msec 2024-11-27T01:14:38,316 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/0caf01005a5fb704a147ee46c7227b3e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,318 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 0caf01005a5fb704a147ee46c7227b3e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66293652, jitterRate=-0.012147605419158936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,318 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:38,318 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 0caf01005a5fb704a147ee46c7227b3e: Running coprocessor pre-open hook at 1732670078221Writing region info on filesystem at 1732670078221Initializing all the Stores at 1732670078245 (+24 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078245Cleaning up temporary data from old regions at 1732670078280 (+35 ms)Running coprocessor post-open hooks at 1732670078318 (+38 ms)Region opened successfully at 1732670078318 2024-11-27T01:14:38,319 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e., pid=36, masterSystemTime=1732670078172 2024-11-27T01:14:38,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8695d5f6c29ed614cc7e97c2ba4c6724, ASSIGN in 567 msec 2024-11-27T01:14:38,321 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d17b306f903adc2b73f818b128d5cd67, ASSIGN 2024-11-27T01:14:38,322 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d17b306f903adc2b73f818b128d5cd67, ASSIGN; state=OFFLINE, location=7d6f26a4c136,40647,1732670073826; forceNewPlan=false, retain=false 2024-11-27T01:14:38,322 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:38,322 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:38,323 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:38,323 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 335860f4847882dab27e1db49323c60b, NAME => 'test,xxx,1732670076899.335860f4847882dab27e1db49323c60b.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-27T01:14:38,323 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,323 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,xxx,1732670076899.335860f4847882dab27e1db49323c60b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,324 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,324 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,325 INFO [StoreOpener-603677e75f480cfd74204fce51bd5359-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,326 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4a61d50f100ead183465a200984a5e77/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=0caf01005a5fb704a147ee46c7227b3e, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,327 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 4a61d50f100ead183465a200984a5e77; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65191875, jitterRate=-0.028565362095832825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,327 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:38,327 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 4a61d50f100ead183465a200984a5e77: Running coprocessor pre-open hook at 1732670078289Writing region info on filesystem at 1732670078289Initializing all the Stores at 1732670078292 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078292Cleaning up temporary data from old regions at 1732670078312 (+20 ms)Running coprocessor post-open hooks at 1732670078327 (+15 ms)Region opened successfully at 1732670078327 2024-11-27T01:14:38,328 INFO [StoreOpener-335860f4847882dab27e1db49323c60b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,333 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77., pid=34, masterSystemTime=1732670078164 2024-11-27T01:14:38,335 INFO [StoreOpener-335860f4847882dab27e1db49323c60b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335860f4847882dab27e1db49323c60b columnFamilyName family 2024-11-27T01:14:38,335 DEBUG [StoreOpener-335860f4847882dab27e1db49323c60b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,32943,1732670073882 because future has completed 2024-11-27T01:14:38,338 INFO [StoreOpener-335860f4847882dab27e1db49323c60b-1 {}] regionserver.HStore(327): Store=335860f4847882dab27e1db49323c60b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,338 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,339 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:38,339 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:38,339 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:38,339 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => edc306f98fb391a3fdc7623063a1cbf8, NAME => 'test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-27T01:14:38,340 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,340 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,340 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,340 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,341 INFO [StoreOpener-603677e75f480cfd74204fce51bd5359-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603677e75f480cfd74204fce51bd5359 columnFamilyName family 2024-11-27T01:14:38,342 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=4a61d50f100ead183465a200984a5e77, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:38,342 DEBUG [StoreOpener-603677e75f480cfd74204fce51bd5359-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,343 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,343 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,345 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7d6f26a4c136,32943,1732670073882, table=test, region=0caf01005a5fb704a147ee46c7227b3e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T01:14:38,348 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,348 INFO [StoreOpener-603677e75f480cfd74204fce51bd5359-1 {}] regionserver.HStore(327): Store=603677e75f480cfd74204fce51bd5359/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,348 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,349 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a61d50f100ead183465a200984a5e77, server=7d6f26a4c136,39915,1732670073690 because future has completed 2024-11-27T01:14:38,350 INFO [StoreOpener-edc306f98fb391a3fdc7623063a1cbf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,350 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,351 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,351 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,354 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,354 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,356 INFO [StoreOpener-edc306f98fb391a3fdc7623063a1cbf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edc306f98fb391a3fdc7623063a1cbf8 columnFamilyName family 2024-11-27T01:14:38,356 DEBUG [StoreOpener-edc306f98fb391a3fdc7623063a1cbf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,357 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,357 INFO [StoreOpener-edc306f98fb391a3fdc7623063a1cbf8-1 {}] regionserver.HStore(327): Store=edc306f98fb391a3fdc7623063a1cbf8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,358 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,359 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,359 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,360 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,360 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=25 2024-11-27T01:14:38,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,32943,1732670073882 in 320 msec 2024-11-27T01:14:38,364 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN in 614 msec 2024-11-27T01:14:38,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=17 2024-11-27T01:14:38,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a61d50f100ead183465a200984a5e77, server=7d6f26a4c136,39915,1732670073690 in 366 msec 2024-11-27T01:14:38,373 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/335860f4847882dab27e1db49323c60b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,374 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 335860f4847882dab27e1db49323c60b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73742526, jitterRate=0.09884926676750183}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,375 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:38,375 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 335860f4847882dab27e1db49323c60b: Running coprocessor pre-open hook at 1732670078324Writing region info on filesystem at 1732670078324Initializing all the Stores at 1732670078326 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078326Cleaning up temporary data from old regions at 1732670078348 (+22 ms)Running coprocessor post-open hooks at 1732670078375 (+27 ms)Region opened successfully at 1732670078375 2024-11-27T01:14:38,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4a61d50f100ead183465a200984a5e77, ASSIGN in 624 msec 2024-11-27T01:14:38,376 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:38,376 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732670076899.335860f4847882dab27e1db49323c60b., pid=35, masterSystemTime=1732670078172 2024-11-27T01:14:38,378 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN; state=OFFLINE, location=7d6f26a4c136,32943,1732670073882; forceNewPlan=false, retain=false 2024-11-27T01:14:38,379 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:38,379 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:38,379 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:38,379 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 8af3d7357dfa0f8aa36ba6b01663d6f6, NAME => 'test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-27T01:14:38,380 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,380 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,380 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,380 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,382 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/edc306f98fb391a3fdc7623063a1cbf8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,382 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened edc306f98fb391a3fdc7623063a1cbf8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58951347, jitterRate=-0.12155647575855255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,383 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:38,383 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for edc306f98fb391a3fdc7623063a1cbf8: Running coprocessor pre-open hook at 1732670078340Writing region info on filesystem at 1732670078340Initializing all the Stores at 1732670078344 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078344Cleaning up temporary data from old regions at 1732670078360 (+16 ms)Running coprocessor post-open hooks at 1732670078383 (+23 ms)Region opened successfully at 1732670078383 2024-11-27T01:14:38,384 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8., pid=39, masterSystemTime=1732670078164 2024-11-27T01:14:38,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=335860f4847882dab27e1db49323c60b, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,387 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3cfaabbc55e590863a7ecf45872977b3, ASSIGN 2024-11-27T01:14:38,389 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3cfaabbc55e590863a7ecf45872977b3, ASSIGN; state=OFFLINE, location=7d6f26a4c136,39915,1732670073690; forceNewPlan=false, retain=false 2024-11-27T01:14:38,389 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/603677e75f480cfd74204fce51bd5359/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,389 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:38,389 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:38,390 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:38,390 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 79f1ff709eae178b11e9bcfa1e4131b2, NAME => 'test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-27T01:14:38,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,32943,1732670073882 because future has completed 2024-11-27T01:14:38,390 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,390 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,390 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,390 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,391 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 603677e75f480cfd74204fce51bd5359; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61509665, jitterRate=-0.0834345668554306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,391 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:38,391 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 603677e75f480cfd74204fce51bd5359: Running coprocessor pre-open hook at 1732670078290Writing region info on filesystem at 1732670078290Initializing all the Stores at 1732670078293 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078293Cleaning up temporary data from old regions at 1732670078354 (+61 ms)Running coprocessor post-open hooks at 1732670078391 (+37 ms)Region opened successfully at 1732670078391 2024-11-27T01:14:38,393 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732670076899.603677e75f480cfd74204fce51bd5359., pid=40, masterSystemTime=1732670078121 2024-11-27T01:14:38,396 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:38,396 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:38,396 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:38,396 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => b272925d8b30354bd56e190c0c041139, NAME => 'test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-27T01:14:38,397 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,397 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,397 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,397 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=28 2024-11-27T01:14:38,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,32943,1732670073882 in 382 msec 2024-11-27T01:14:38,405 INFO [StoreOpener-8af3d7357dfa0f8aa36ba6b01663d6f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,408 INFO [StoreOpener-8af3d7357dfa0f8aa36ba6b01663d6f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8af3d7357dfa0f8aa36ba6b01663d6f6 columnFamilyName family 2024-11-27T01:14:38,408 DEBUG [StoreOpener-8af3d7357dfa0f8aa36ba6b01663d6f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,408 INFO [StoreOpener-79f1ff709eae178b11e9bcfa1e4131b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,409 INFO [StoreOpener-8af3d7357dfa0f8aa36ba6b01663d6f6-1 {}] regionserver.HStore(327): Store=8af3d7357dfa0f8aa36ba6b01663d6f6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,409 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,411 INFO [StoreOpener-79f1ff709eae178b11e9bcfa1e4131b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79f1ff709eae178b11e9bcfa1e4131b2 columnFamilyName family 2024-11-27T01:14:38,411 DEBUG [StoreOpener-79f1ff709eae178b11e9bcfa1e4131b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,413 INFO [StoreOpener-b272925d8b30354bd56e190c0c041139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,414 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,415 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN in 656 msec 2024-11-27T01:14:38,417 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,417 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,418 INFO [StoreOpener-79f1ff709eae178b11e9bcfa1e4131b2-1 {}] regionserver.HStore(327): Store=79f1ff709eae178b11e9bcfa1e4131b2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,418 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,419 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=603677e75f480cfd74204fce51bd5359, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,420 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,420 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,421 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,421 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,422 INFO [StoreOpener-b272925d8b30354bd56e190c0c041139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b272925d8b30354bd56e190c0c041139 columnFamilyName family 2024-11-27T01:14:38,422 DEBUG [StoreOpener-b272925d8b30354bd56e190c0c041139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 603677e75f480cfd74204fce51bd5359, server=7d6f26a4c136,40647,1732670073826 because future has completed 2024-11-27T01:14:38,425 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:38,426 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,424 INFO [StoreOpener-b272925d8b30354bd56e190c0c041139-1 {}] regionserver.HStore(327): Store=b272925d8b30354bd56e190c0c041139/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,427 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,428 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc306f98fb391a3fdc7623063a1cbf8, server=7d6f26a4c136,39915,1732670073690 because future has completed 2024-11-27T01:14:38,429 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,430 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,430 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,433 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,435 INFO [7d6f26a4c136:33239 {}] balancer.BaseLoadBalancer(388): Reassigned 4 regions. 4 retained the pre-restart assignment. 2024-11-27T01:14:38,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=9 2024-11-27T01:14:38,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 603677e75f480cfd74204fce51bd5359, server=7d6f26a4c136,40647,1732670073826 in 348 msec 2024-11-27T01:14:38,438 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8af3d7357dfa0f8aa36ba6b01663d6f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,438 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 8af3d7357dfa0f8aa36ba6b01663d6f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66503429, jitterRate=-0.009021684527397156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,439 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:38,439 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 8af3d7357dfa0f8aa36ba6b01663d6f6: Running coprocessor pre-open hook at 1732670078380Writing region info on filesystem at 1732670078380Initializing all the Stores at 1732670078382 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078382Cleaning up temporary data from old regions at 1732670078417 (+35 ms)Running coprocessor post-open hooks at 1732670078439 (+22 ms)Region opened successfully at 1732670078439 2024-11-27T01:14:38,442 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7d6f26a4c136,39915,1732670073690, table=test, region=edc306f98fb391a3fdc7623063a1cbf8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T01:14:38,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=603677e75f480cfd74204fce51bd5359, ASSIGN in 690 msec 2024-11-27T01:14:38,442 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6., pid=52, masterSystemTime=1732670078331 2024-11-27T01:14:38,443 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d17b306f903adc2b73f818b128d5cd67, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,445 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/b272925d8b30354bd56e190c0c041139/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,446 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:38,447 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:38,447 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened b272925d8b30354bd56e190c0c041139; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72465005, jitterRate=0.07981272041797638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,447 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:38,447 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:38,447 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for b272925d8b30354bd56e190c0c041139: Running coprocessor pre-open hook at 1732670078397Writing region info on filesystem at 1732670078397Initializing all the Stores at 1732670078399 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078399Cleaning up temporary data from old regions at 1732670078430 (+31 ms)Running coprocessor post-open hooks at 1732670078447 (+17 ms)Region opened successfully at 1732670078447 2024-11-27T01:14:38,447 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ae4c3f2c0fe2fac2f19395fca402eb5, NAME => 'test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-27T01:14:38,447 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,448 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,448 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,448 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139., pid=45, masterSystemTime=1732670078268 2024-11-27T01:14:38,448 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,453 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:38,453 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:38,453 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:38,453 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fed3d0dc5a44acae9cbb7359cdf2051, NAME => 'test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-27T01:14:38,454 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,454 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,454 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,454 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d17b306f903adc2b73f818b128d5cd67, ASSIGN because future has completed 2024-11-27T01:14:38,457 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/79f1ff709eae178b11e9bcfa1e4131b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,457 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,458 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 79f1ff709eae178b11e9bcfa1e4131b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72518055, jitterRate=0.08060322701931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,458 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:38,458 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 79f1ff709eae178b11e9bcfa1e4131b2: Running coprocessor pre-open hook at 1732670078391Writing region info on filesystem at 1732670078391Initializing all the Stores at 1732670078393 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078393Cleaning up temporary data from old regions at 1732670078421 (+28 ms)Running coprocessor post-open hooks at 1732670078458 (+37 ms)Region opened successfully at 1732670078458 2024-11-27T01:14:38,459 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2., pid=37, masterSystemTime=1732670078164 2024-11-27T01:14:38,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d17b306f903adc2b73f818b128d5cd67, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:38,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,32943,1732670073882 because future has completed 2024-11-27T01:14:38,462 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:38,462 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:38,463 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:38,463 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => b85ee4f1fa33a9cbdcb45c32b6176c88, NAME => 'test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-27T01:14:38,463 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,463 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,463 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,464 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,468 INFO [StoreOpener-2fed3d0dc5a44acae9cbb7359cdf2051-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,471 INFO [StoreOpener-2fed3d0dc5a44acae9cbb7359cdf2051-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fed3d0dc5a44acae9cbb7359cdf2051 columnFamilyName family 2024-11-27T01:14:38,472 DEBUG [StoreOpener-2fed3d0dc5a44acae9cbb7359cdf2051-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,473 INFO [StoreOpener-2fed3d0dc5a44acae9cbb7359cdf2051-1 {}] regionserver.HStore(327): Store=2fed3d0dc5a44acae9cbb7359cdf2051/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,473 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,474 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,476 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,476 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,477 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=18 2024-11-27T01:14:38,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,32943,1732670073882 in 220 msec 2024-11-27T01:14:38,479 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN in 730 msec 2024-11-27T01:14:38,483 INFO [StoreOpener-8ae4c3f2c0fe2fac2f19395fca402eb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,485 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:38,486 INFO [StoreOpener-8ae4c3f2c0fe2fac2f19395fca402eb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ae4c3f2c0fe2fac2f19395fca402eb5 columnFamilyName family 2024-11-27T01:14:38,486 DEBUG [StoreOpener-8ae4c3f2c0fe2fac2f19395fca402eb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,487 INFO [StoreOpener-8ae4c3f2c0fe2fac2f19395fca402eb5-1 {}] regionserver.HStore(327): Store=8ae4c3f2c0fe2fac2f19395fca402eb5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,487 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,488 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,489 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79f1ff709eae178b11e9bcfa1e4131b2, server=7d6f26a4c136,39915,1732670073690 because future has completed 2024-11-27T01:14:38,490 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,490 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,491 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=b272925d8b30354bd56e190c0c041139, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,494 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure b272925d8b30354bd56e190c0c041139, server=7d6f26a4c136,40647,1732670073826 because future has completed 2024-11-27T01:14:38,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=11 2024-11-27T01:14:38,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 79f1ff709eae178b11e9bcfa1e4131b2, server=7d6f26a4c136,39915,1732670073690 in 451 msec 2024-11-27T01:14:38,503 INFO [StoreOpener-b85ee4f1fa33a9cbdcb45c32b6176c88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,505 INFO [StoreOpener-b85ee4f1fa33a9cbdcb45c32b6176c88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85ee4f1fa33a9cbdcb45c32b6176c88 columnFamilyName family 2024-11-27T01:14:38,505 DEBUG [StoreOpener-b85ee4f1fa33a9cbdcb45c32b6176c88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,505 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8ae4c3f2c0fe2fac2f19395fca402eb5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79f1ff709eae178b11e9bcfa1e4131b2, ASSIGN in 753 msec 2024-11-27T01:14:38,506 INFO [StoreOpener-b85ee4f1fa33a9cbdcb45c32b6176c88-1 {}] regionserver.HStore(327): Store=b85ee4f1fa33a9cbdcb45c32b6176c88/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,506 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,506 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 8ae4c3f2c0fe2fac2f19395fca402eb5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69297900, jitterRate=0.0326191782951355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,506 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:38,507 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 8ae4c3f2c0fe2fac2f19395fca402eb5: Running coprocessor pre-open hook at 1732670078449Writing region info on filesystem at 1732670078449Initializing all the Stores at 1732670078451 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078451Cleaning up temporary data from old regions at 1732670078491 (+40 ms)Running coprocessor post-open hooks at 1732670078506 (+15 ms)Region opened successfully at 1732670078506 2024-11-27T01:14:38,507 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/2fed3d0dc5a44acae9cbb7359cdf2051/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,507 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,508 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,508 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 2fed3d0dc5a44acae9cbb7359cdf2051; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62244064, jitterRate=-0.07249116897583008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,508 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:38,508 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 2fed3d0dc5a44acae9cbb7359cdf2051: Running coprocessor pre-open hook at 1732670078454Writing region info on filesystem at 1732670078454Initializing all the Stores at 1732670078456 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078456Cleaning up temporary data from old regions at 1732670078477 (+21 ms)Running coprocessor post-open hooks at 1732670078508 (+31 ms)Region opened successfully at 1732670078508 2024-11-27T01:14:38,509 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,509 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,509 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051., pid=43, masterSystemTime=1732670078268 2024-11-27T01:14:38,510 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5., pid=51, masterSystemTime=1732670078331 2024-11-27T01:14:38,512 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,514 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:38,514 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:38,514 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:38,514 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 1229242f433b46943919d6643a95d4a4, NAME => 'test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-27T01:14:38,515 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,515 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,515 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,515 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=19 2024-11-27T01:14:38,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure b272925d8b30354bd56e190c0c041139, server=7d6f26a4c136,40647,1732670073826 in 379 msec 2024-11-27T01:14:38,518 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:38,518 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:38,518 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:38,518 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => a05cacf8eaf88a4edb7255a00c86d78c, NAME => 'test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-27T01:14:38,519 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,519 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,519 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,519 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b272925d8b30354bd56e190c0c041139, ASSIGN in 769 msec 2024-11-27T01:14:38,524 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,526 INFO [StoreOpener-1229242f433b46943919d6643a95d4a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,528 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/b85ee4f1fa33a9cbdcb45c32b6176c88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,32943,1732670073882 because future has completed 2024-11-27T01:14:38,529 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened b85ee4f1fa33a9cbdcb45c32b6176c88; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67054903, jitterRate=-8.040815591812134E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,530 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:38,530 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for b85ee4f1fa33a9cbdcb45c32b6176c88: Running coprocessor pre-open hook at 1732670078464Writing region info on filesystem at 1732670078464Initializing all the Stores at 1732670078465 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078465Cleaning up temporary data from old regions at 1732670078509 (+44 ms)Running coprocessor post-open hooks at 1732670078530 (+21 ms)Region opened successfully at 1732670078530 2024-11-27T01:14:38,530 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,531 INFO [StoreOpener-a05cacf8eaf88a4edb7255a00c86d78c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,531 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88., pid=41, masterSystemTime=1732670078164 2024-11-27T01:14:38,534 INFO [StoreOpener-a05cacf8eaf88a4edb7255a00c86d78c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a05cacf8eaf88a4edb7255a00c86d78c columnFamilyName family 2024-11-27T01:14:38,534 DEBUG [StoreOpener-a05cacf8eaf88a4edb7255a00c86d78c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,536 INFO [StoreOpener-a05cacf8eaf88a4edb7255a00c86d78c-1 {}] regionserver.HStore(327): Store=a05cacf8eaf88a4edb7255a00c86d78c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,536 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,536 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:38,536 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:38,536 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:38,537 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => eedb1cdc1fad3f780344bbfed739194f, NAME => 'test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-27T01:14:38,537 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,537 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,537 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,537 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,537 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fed3d0dc5a44acae9cbb7359cdf2051, server=7d6f26a4c136,40647,1732670073826 because future has completed 2024-11-27T01:14:38,540 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:38,541 INFO [StoreOpener-1229242f433b46943919d6643a95d4a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1229242f433b46943919d6643a95d4a4 columnFamilyName family 2024-11-27T01:14:38,541 DEBUG [StoreOpener-1229242f433b46943919d6643a95d4a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,542 INFO [StoreOpener-1229242f433b46943919d6643a95d4a4-1 {}] regionserver.HStore(327): Store=1229242f433b46943919d6643a95d4a4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,543 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,544 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,544 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure b85ee4f1fa33a9cbdcb45c32b6176c88, server=7d6f26a4c136,39915,1732670073690 because future has completed 2024-11-27T01:14:38,545 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,545 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,545 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,545 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,545 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,546 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7d6f26a4c136,39915,1732670073690, table=test, region=edc306f98fb391a3fdc7623063a1cbf8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T01:14:38,546 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7d6f26a4c136,39915,1732670073690, table=test, region=b85ee4f1fa33a9cbdcb45c32b6176c88. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T01:14:38,548 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,550 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=13 2024-11-27T01:14:38,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fed3d0dc5a44acae9cbb7359cdf2051, server=7d6f26a4c136,40647,1732670073826 in 433 msec 2024-11-27T01:14:38,556 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7d6f26a4c136,32943,1732670073882, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T01:14:38,558 INFO [StoreOpener-eedb1cdc1fad3f780344bbfed739194f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,560 INFO [StoreOpener-eedb1cdc1fad3f780344bbfed739194f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eedb1cdc1fad3f780344bbfed739194f columnFamilyName family 2024-11-27T01:14:38,560 DEBUG [StoreOpener-eedb1cdc1fad3f780344bbfed739194f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,561 INFO [StoreOpener-eedb1cdc1fad3f780344bbfed739194f-1 {}] regionserver.HStore(327): Store=eedb1cdc1fad3f780344bbfed739194f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2fed3d0dc5a44acae9cbb7359cdf2051, ASSIGN in 806 msec 2024-11-27T01:14:38,561 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,563 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,563 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,564 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,564 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,567 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=23 2024-11-27T01:14:38,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure b85ee4f1fa33a9cbdcb45c32b6176c88, server=7d6f26a4c136,39915,1732670073690 in 471 msec 2024-11-27T01:14:38,570 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/1229242f433b46943919d6643a95d4a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b85ee4f1fa33a9cbdcb45c32b6176c88, ASSIGN in 820 msec 2024-11-27T01:14:38,571 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 1229242f433b46943919d6643a95d4a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69111636, jitterRate=0.029843628406524658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,572 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:38,572 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 1229242f433b46943919d6643a95d4a4: Running coprocessor pre-open hook at 1732670078515Writing region info on filesystem at 1732670078515Initializing all the Stores at 1732670078517 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078517Cleaning up temporary data from old regions at 1732670078545 (+28 ms)Running coprocessor post-open hooks at 1732670078572 (+27 ms)Region opened successfully at 1732670078572 2024-11-27T01:14:38,575 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4., pid=46, masterSystemTime=1732670078268 2024-11-27T01:14:38,578 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/a05cacf8eaf88a4edb7255a00c86d78c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,578 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:38,578 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:38,578 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened a05cacf8eaf88a4edb7255a00c86d78c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61845983, jitterRate=-0.07842303812503815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,578 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:38,579 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:38,579 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for a05cacf8eaf88a4edb7255a00c86d78c: Running coprocessor pre-open hook at 1732670078519Writing region info on filesystem at 1732670078519Initializing all the Stores at 1732670078521 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078521Cleaning up temporary data from old regions at 1732670078545 (+24 ms)Running coprocessor post-open hooks at 1732670078579 (+34 ms)Region opened successfully at 1732670078579 2024-11-27T01:14:38,579 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 796f7fc8fad8d003c466a3f53b960815, NAME => 'test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-27T01:14:38,579 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,579 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,580 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,580 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,580 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c., pid=50, masterSystemTime=1732670078331 2024-11-27T01:14:38,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=16 2024-11-27T01:14:38,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,32943,1732670073882 in 353 msec 2024-11-27T01:14:38,583 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:38,583 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:38,583 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:38,583 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e92cc2f48378cc4a64c15f36bcf6052, NAME => 'test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-27T01:14:38,584 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,584 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,584 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,584 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,585 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,588 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/eedb1cdc1fad3f780344bbfed739194f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,32943,1732670073882 because future has completed 2024-11-27T01:14:38,589 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened eedb1cdc1fad3f780344bbfed739194f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62055825, jitterRate=-0.07529614865779877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,589 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:38,589 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for eedb1cdc1fad3f780344bbfed739194f: Running coprocessor pre-open hook at 1732670078537Writing region info on filesystem at 1732670078537Initializing all the Stores at 1732670078540 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078541 (+1 ms)Cleaning up temporary data from old regions at 1732670078564 (+23 ms)Running coprocessor post-open hooks at 1732670078589 (+25 ms)Region opened successfully at 1732670078589 2024-11-27T01:14:38,591 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f., pid=38, masterSystemTime=1732670078164 2024-11-27T01:14:38,596 INFO [StoreOpener-796f7fc8fad8d003c466a3f53b960815-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,600 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:38,600 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:38,600 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:38,600 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => fc23d7fa863cb970ac753db9e4e26b26, NAME => 'test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-27T01:14:38,601 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,601 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,601 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,601 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,602 INFO [StoreOpener-8e92cc2f48378cc4a64c15f36bcf6052-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,604 INFO [StoreOpener-796f7fc8fad8d003c466a3f53b960815-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 796f7fc8fad8d003c466a3f53b960815 columnFamilyName family 2024-11-27T01:14:38,604 INFO [StoreOpener-fc23d7fa863cb970ac753db9e4e26b26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=12 2024-11-27T01:14:38,604 DEBUG [StoreOpener-796f7fc8fad8d003c466a3f53b960815-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,32943,1732670073882 in 390 msec 2024-11-27T01:14:38,605 INFO [StoreOpener-796f7fc8fad8d003c466a3f53b960815-1 {}] regionserver.HStore(327): Store=796f7fc8fad8d003c466a3f53b960815/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,605 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,606 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,607 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,607 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,607 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN in 857 msec 2024-11-27T01:14:38,608 INFO [StoreOpener-fc23d7fa863cb970ac753db9e4e26b26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc23d7fa863cb970ac753db9e4e26b26 columnFamilyName family 2024-11-27T01:14:38,609 DEBUG [StoreOpener-fc23d7fa863cb970ac753db9e4e26b26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,610 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:38,610 INFO [StoreOpener-fc23d7fa863cb970ac753db9e4e26b26-1 {}] regionserver.HStore(327): Store=fc23d7fa863cb970ac753db9e4e26b26/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,610 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,611 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,611 INFO [StoreOpener-8e92cc2f48378cc4a64c15f36bcf6052-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e92cc2f48378cc4a64c15f36bcf6052 columnFamilyName family 2024-11-27T01:14:38,611 DEBUG [StoreOpener-8e92cc2f48378cc4a64c15f36bcf6052-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,611 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,612 INFO [StoreOpener-8e92cc2f48378cc4a64c15f36bcf6052-1 {}] regionserver.HStore(327): Store=8e92cc2f48378cc4a64c15f36bcf6052/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,612 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,612 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,613 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,613 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,614 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,615 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure eedb1cdc1fad3f780344bbfed739194f, server=7d6f26a4c136,39915,1732670073690 because future has completed 2024-11-27T01:14:38,616 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,616 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,619 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN in 864 msec 2024-11-27T01:14:38,624 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,625 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/796f7fc8fad8d003c466a3f53b960815/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,642 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8e92cc2f48378cc4a64c15f36bcf6052/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,642 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 796f7fc8fad8d003c466a3f53b960815; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59408274, jitterRate=-0.11474773287773132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,642 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:38,642 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 796f7fc8fad8d003c466a3f53b960815: Running coprocessor pre-open hook at 1732670078580Writing region info on filesystem at 1732670078580Initializing all the Stores at 1732670078582 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078582Cleaning up temporary data from old regions at 1732670078608 (+26 ms)Running coprocessor post-open hooks at 1732670078642 (+34 ms)Region opened successfully at 1732670078642 2024-11-27T01:14:38,643 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 8e92cc2f48378cc4a64c15f36bcf6052; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62674845, jitterRate=-0.06607203185558319}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,643 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:38,643 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 8e92cc2f48378cc4a64c15f36bcf6052: Running coprocessor pre-open hook at 1732670078584Writing region info on filesystem at 1732670078584Initializing all the Stores at 1732670078585 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078585Cleaning up temporary data from old regions at 1732670078616 (+31 ms)Running coprocessor post-open hooks at 1732670078643 (+27 ms)Region opened successfully at 1732670078643 2024-11-27T01:14:38,644 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815., pid=44, masterSystemTime=1732670078268 2024-11-27T01:14:38,645 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052., pid=49, masterSystemTime=1732670078331 2024-11-27T01:14:38,645 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/fc23d7fa863cb970ac753db9e4e26b26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,646 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened fc23d7fa863cb970ac753db9e4e26b26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60160002, jitterRate=-0.10354611277580261}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,646 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:38,647 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for fc23d7fa863cb970ac753db9e4e26b26: Running coprocessor pre-open hook at 1732670078601Writing region info on filesystem at 1732670078601Initializing all the Stores at 1732670078603 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078603Cleaning up temporary data from old regions at 1732670078613 (+10 ms)Running coprocessor post-open hooks at 1732670078646 (+33 ms)Region opened successfully at 1732670078647 (+1 ms) 2024-11-27T01:14:38,648 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26., pid=42, masterSystemTime=1732670078164 2024-11-27T01:14:38,648 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:38,649 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:38,649 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:38,649 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => ff2b8d760c6f5acc07476412531886a0, NAME => 'test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-27T01:14:38,649 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,649 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,650 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,650 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=14 2024-11-27T01:14:38,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure eedb1cdc1fad3f780344bbfed739194f, server=7d6f26a4c136,39915,1732670073690 in 579 msec 2024-11-27T01:14:38,652 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:38,652 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:38,652 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:38,653 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b3cb4626ebbddc356f324f4faf22f80, NAME => 'test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-27T01:14:38,653 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,653 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,653 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,653 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,653 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:38,653 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:38,654 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7d6f26a4c136,39915,1732670073690, table=test, region=edc306f98fb391a3fdc7623063a1cbf8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T01:14:38,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eedb1cdc1fad3f780344bbfed739194f, ASSIGN in 903 msec 2024-11-27T01:14:38,656 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:38,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc23d7fa863cb970ac753db9e4e26b26, server=7d6f26a4c136,39915,1732670073690 because future has completed 2024-11-27T01:14:38,668 INFO [StoreOpener-ff2b8d760c6f5acc07476412531886a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,670 INFO [StoreOpener-4b3cb4626ebbddc356f324f4faf22f80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,673 INFO [StoreOpener-ff2b8d760c6f5acc07476412531886a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff2b8d760c6f5acc07476412531886a0 columnFamilyName family 2024-11-27T01:14:38,673 DEBUG [StoreOpener-ff2b8d760c6f5acc07476412531886a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,674 INFO [StoreOpener-ff2b8d760c6f5acc07476412531886a0-1 {}] regionserver.HStore(327): Store=ff2b8d760c6f5acc07476412531886a0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,674 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,675 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,676 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,676 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,676 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,679 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=26 2024-11-27T01:14:38,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure fc23d7fa863cb970ac753db9e4e26b26, server=7d6f26a4c136,39915,1732670073690 in 565 msec 2024-11-27T01:14:38,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fc23d7fa863cb970ac753db9e4e26b26, ASSIGN in 933 msec 2024-11-27T01:14:38,686 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e92cc2f48378cc4a64c15f36bcf6052, server=7d6f26a4c136,32943,1732670073882 because future has completed 2024-11-27T01:14:38,690 INFO [StoreOpener-4b3cb4626ebbddc356f324f4faf22f80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b3cb4626ebbddc356f324f4faf22f80 columnFamilyName family 2024-11-27T01:14:38,690 DEBUG [StoreOpener-4b3cb4626ebbddc356f324f4faf22f80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,691 INFO [StoreOpener-4b3cb4626ebbddc356f324f4faf22f80-1 {}] regionserver.HStore(327): Store=4b3cb4626ebbddc356f324f4faf22f80/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,691 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,692 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,693 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,694 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,694 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,694 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/ff2b8d760c6f5acc07476412531886a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 796f7fc8fad8d003c466a3f53b960815, server=7d6f26a4c136,40647,1732670073826 because future has completed 2024-11-27T01:14:38,696 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened ff2b8d760c6f5acc07476412531886a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67126877, jitterRate=2.6841461658477783E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,696 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:38,696 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for ff2b8d760c6f5acc07476412531886a0: Running coprocessor pre-open hook at 1732670078650Writing region info on filesystem at 1732670078650Initializing all the Stores at 1732670078654 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078654Cleaning up temporary data from old regions at 1732670078677 (+23 ms)Running coprocessor post-open hooks at 1732670078696 (+19 ms)Region opened successfully at 1732670078696 2024-11-27T01:14:38,696 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,697 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0., pid=47, masterSystemTime=1732670078268 2024-11-27T01:14:38,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=10 2024-11-27T01:14:38,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e92cc2f48378cc4a64c15f36bcf6052, server=7d6f26a4c136,32943,1732670073882 in 503 msec 2024-11-27T01:14:38,701 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:38,701 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:38,701 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:38,701 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => d17b306f903adc2b73f818b128d5cd67, NAME => 'test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-27T01:14:38,702 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,702 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,702 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,702 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,704 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff2b8d760c6f5acc07476412531886a0, server=7d6f26a4c136,40647,1732670073826 because future has completed 2024-11-27T01:14:38,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN in 959 msec 2024-11-27T01:14:38,713 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4b3cb4626ebbddc356f324f4faf22f80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,714 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 4b3cb4626ebbddc356f324f4faf22f80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60433734, jitterRate=-0.0994671881198883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,714 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:38,714 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 4b3cb4626ebbddc356f324f4faf22f80: Running coprocessor pre-open hook at 1732670078653Writing region info on filesystem at 1732670078653Initializing all the Stores at 1732670078655 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078655Cleaning up temporary data from old regions at 1732670078694 (+39 ms)Running coprocessor post-open hooks at 1732670078714 (+20 ms)Region opened successfully at 1732670078714 2024-11-27T01:14:38,715 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80., pid=48, masterSystemTime=1732670078331 2024-11-27T01:14:38,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=27 2024-11-27T01:14:38,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure ff2b8d760c6f5acc07476412531886a0, server=7d6f26a4c136,40647,1732670073826 in 558 msec 2024-11-27T01:14:38,718 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:38,719 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:38,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ff2b8d760c6f5acc07476412531886a0, ASSIGN in 969 msec 2024-11-27T01:14:38,721 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,721 INFO [StoreOpener-d17b306f903adc2b73f818b128d5cd67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,724 INFO [StoreOpener-d17b306f903adc2b73f818b128d5cd67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d17b306f903adc2b73f818b128d5cd67 columnFamilyName family 2024-11-27T01:14:38,724 DEBUG [StoreOpener-d17b306f903adc2b73f818b128d5cd67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,32943,1732670073882 because future has completed 2024-11-27T01:14:38,728 INFO [StoreOpener-d17b306f903adc2b73f818b128d5cd67-1 {}] regionserver.HStore(327): Store=d17b306f903adc2b73f818b128d5cd67/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,728 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,729 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,730 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,731 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,731 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=15 2024-11-27T01:14:38,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 796f7fc8fad8d003c466a3f53b960815, server=7d6f26a4c136,40647,1732670073826 in 607 msec 2024-11-27T01:14:38,733 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=796f7fc8fad8d003c466a3f53b960815, ASSIGN in 984 msec 2024-11-27T01:14:38,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=22 2024-11-27T01:14:38,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,32943,1732670073882 in 565 msec 2024-11-27T01:14:38,745 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/d17b306f903adc2b73f818b128d5cd67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN in 992 msec 2024-11-27T01:14:38,746 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened d17b306f903adc2b73f818b128d5cd67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60109582, jitterRate=-0.10429742932319641}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,746 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:38,746 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for d17b306f903adc2b73f818b128d5cd67: Running coprocessor pre-open hook at 1732670078702Writing region info on filesystem at 1732670078702Initializing all the Stores at 1732670078703 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078704 (+1 ms)Cleaning up temporary data from old regions at 1732670078731 (+27 ms)Running coprocessor post-open hooks at 1732670078746 (+15 ms)Region opened successfully at 1732670078746 2024-11-27T01:14:38,748 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67., pid=53, masterSystemTime=1732670078617 2024-11-27T01:14:38,748 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1229242f433b46943919d6643a95d4a4, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,751 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:38,751 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:38,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1229242f433b46943919d6643a95d4a4, server=7d6f26a4c136,40647,1732670073826 because future has completed 2024-11-27T01:14:38,753 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:38,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d17b306f903adc2b73f818b128d5cd67, server=7d6f26a4c136,40647,1732670073826 because future has completed 2024-11-27T01:14:38,759 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7d6f26a4c136,39915,1732670073690, table=test, region=edc306f98fb391a3fdc7623063a1cbf8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T01:14:38,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=24 2024-11-27T01:14:38,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 1229242f433b46943919d6643a95d4a4, server=7d6f26a4c136,40647,1732670073826 in 613 msec 2024-11-27T01:14:38,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1229242f433b46943919d6643a95d4a4, ASSIGN in 1.0130 sec 2024-11-27T01:14:38,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-11-27T01:14:38,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure d17b306f903adc2b73f818b128d5cd67, server=7d6f26a4c136,40647,1732670073826 in 305 msec 2024-11-27T01:14:38,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d17b306f903adc2b73f818b128d5cd67, ASSIGN in 1.0220 sec 2024-11-27T01:14:38,772 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=968f86830e42ee358d5e11672847740f, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:38,773 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3cfaabbc55e590863a7ecf45872977b3, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:38,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=968f86830e42ee358d5e11672847740f, ASSIGN because future has completed 2024-11-27T01:14:38,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3cfaabbc55e590863a7ecf45872977b3, ASSIGN because future has completed 2024-11-27T01:14:38,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN because future has completed 2024-11-27T01:14:38,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=21 2024-11-27T01:14:38,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure edc306f98fb391a3fdc7623063a1cbf8, server=7d6f26a4c136,39915,1732670073690 in 704 msec 2024-11-27T01:14:38,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=edc306f98fb391a3fdc7623063a1cbf8, ASSIGN in 1.0340 sec 2024-11-27T01:14:38,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,32943,1732670073882}] 2024-11-27T01:14:38,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cfaabbc55e590863a7ecf45872977b3, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:38,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 968f86830e42ee358d5e11672847740f, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:38,949 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:38,949 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => bbecc4f063e3fd6151247c0507c5bef3, NAME => 'test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-27T01:14:38,949 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,950 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,950 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,950 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,953 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:38,953 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 3cfaabbc55e590863a7ecf45872977b3, NAME => 'test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-27T01:14:38,954 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,954 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:38,954 INFO [StoreOpener-bbecc4f063e3fd6151247c0507c5bef3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,954 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,954 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,956 INFO [StoreOpener-bbecc4f063e3fd6151247c0507c5bef3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbecc4f063e3fd6151247c0507c5bef3 columnFamilyName family 2024-11-27T01:14:38,956 DEBUG [StoreOpener-bbecc4f063e3fd6151247c0507c5bef3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,957 INFO [StoreOpener-bbecc4f063e3fd6151247c0507c5bef3-1 {}] regionserver.HStore(327): Store=bbecc4f063e3fd6151247c0507c5bef3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,957 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,958 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,959 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,959 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,959 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,961 INFO [StoreOpener-3cfaabbc55e590863a7ecf45872977b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,963 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,968 INFO [StoreOpener-3cfaabbc55e590863a7ecf45872977b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cfaabbc55e590863a7ecf45872977b3 columnFamilyName family 2024-11-27T01:14:38,968 DEBUG [StoreOpener-3cfaabbc55e590863a7ecf45872977b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:38,970 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/bbecc4f063e3fd6151247c0507c5bef3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,970 INFO [StoreOpener-3cfaabbc55e590863a7ecf45872977b3-1 {}] regionserver.HStore(327): Store=3cfaabbc55e590863a7ecf45872977b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:38,970 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,970 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened bbecc4f063e3fd6151247c0507c5bef3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63078651, jitterRate=-0.060054853558540344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,970 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:38,971 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for bbecc4f063e3fd6151247c0507c5bef3: Running coprocessor pre-open hook at 1732670078950Writing region info on filesystem at 1732670078950Initializing all the Stores at 1732670078951 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078951Cleaning up temporary data from old regions at 1732670078959 (+8 ms)Running coprocessor post-open hooks at 1732670078970 (+11 ms)Region opened successfully at 1732670078970 2024-11-27T01:14:38,971 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,971 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,972 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,972 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,976 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3., pid=54, masterSystemTime=1732670078941 2024-11-27T01:14:38,978 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,983 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:38,983 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:38,984 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:38,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,32943,1732670073882 because future has completed 2024-11-27T01:14:38,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=6 2024-11-27T01:14:38,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,32943,1732670073882 in 201 msec 2024-11-27T01:14:38,996 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/3cfaabbc55e590863a7ecf45872977b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:38,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN in 1.2470 sec 2024-11-27T01:14:38,997 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 3cfaabbc55e590863a7ecf45872977b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58848159, jitterRate=-0.1230940967798233}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:38,997 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:38,998 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 3cfaabbc55e590863a7ecf45872977b3: Running coprocessor pre-open hook at 1732670078954Writing region info on filesystem at 1732670078954Initializing all the Stores at 1732670078956 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670078956Cleaning up temporary data from old regions at 1732670078972 (+16 ms)Running coprocessor post-open hooks at 1732670078997 (+25 ms)Region opened successfully at 1732670078997 2024-11-27T01:14:38,999 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3., pid=55, masterSystemTime=1732670078947 2024-11-27T01:14:39,002 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:39,002 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:39,003 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:39,003 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 968f86830e42ee358d5e11672847740f, NAME => 'test,ddd,1732670076899.968f86830e42ee358d5e11672847740f.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-27T01:14:39,003 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,003 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,ddd,1732670076899.968f86830e42ee358d5e11672847740f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:39,003 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,004 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,006 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:39,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cfaabbc55e590863a7ecf45872977b3, server=7d6f26a4c136,39915,1732670073690 because future has completed 2024-11-27T01:14:39,013 INFO [StoreOpener-968f86830e42ee358d5e11672847740f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=5 2024-11-27T01:14:39,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3cfaabbc55e590863a7ecf45872977b3, server=7d6f26a4c136,39915,1732670073690 in 219 msec 2024-11-27T01:14:39,018 INFO [StoreOpener-968f86830e42ee358d5e11672847740f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 968f86830e42ee358d5e11672847740f columnFamilyName family 2024-11-27T01:14:39,019 DEBUG [StoreOpener-968f86830e42ee358d5e11672847740f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:39,020 INFO [StoreOpener-968f86830e42ee358d5e11672847740f-1 {}] regionserver.HStore(327): Store=968f86830e42ee358d5e11672847740f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:39,020 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3cfaabbc55e590863a7ecf45872977b3, ASSIGN in 1.2710 sec 2024-11-27T01:14:39,023 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,024 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,025 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,025 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,028 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,033 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/968f86830e42ee358d5e11672847740f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:14:39,034 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 968f86830e42ee358d5e11672847740f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60459009, jitterRate=-0.0990905612707138}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:39,034 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:39,034 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 968f86830e42ee358d5e11672847740f: Running coprocessor pre-open hook at 1732670079004Writing region info on filesystem at 1732670079004Initializing all the Stores at 1732670079006 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670079006Cleaning up temporary data from old regions at 1732670079025 (+19 ms)Running coprocessor post-open hooks at 1732670079034 (+9 ms)Region opened successfully at 1732670079034 2024-11-27T01:14:39,036 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732670076899.968f86830e42ee358d5e11672847740f., pid=56, masterSystemTime=1732670078947 2024-11-27T01:14:39,042 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=968f86830e42ee358d5e11672847740f, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:39,043 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:39,043 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:39,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 968f86830e42ee358d5e11672847740f, server=7d6f26a4c136,39915,1732670073690 because future has completed 2024-11-27T01:14:39,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=8 2024-11-27T01:14:39,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 968f86830e42ee358d5e11672847740f, server=7d6f26a4c136,39915,1732670073690 in 248 msec 2024-11-27T01:14:39,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4 2024-11-27T01:14:39,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=968f86830e42ee358d5e11672847740f, ASSIGN in 1.3050 sec 2024-11-27T01:14:39,059 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T01:14:39,059 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732670079059"}]},"ts":"1732670079059"} 2024-11-27T01:14:39,064 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-27T01:14:39,065 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T01:14:39,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 2.1570 sec 2024-11-27T01:14:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:14:39,086 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-27T01:14:39,086 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-27T01:14:39,088 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:14:39,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-27T01:14:39,118 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:14:39,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-27T01:14:39,120 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:14:39,129 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-27T01:14:39,161 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-27T01:14:39,187 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test', row='', locateType=CURRENT is [region=test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3., hostname=7d6f26a4c136,39915,1732670073690, seqNum=2] 2024-11-27T01:14:39,191 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:14:39,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:14:39,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:14:39,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:14:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bbecc4f063e3fd6151247c0507c5bef3 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8e92cc2f48378cc4a64c15f36bcf6052 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a05cacf8eaf88a4edb7255a00c86d78c changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8ae4c3f2c0fe2fac2f19395fca402eb5 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8af3d7357dfa0f8aa36ba6b01663d6f6 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4b3cb4626ebbddc356f324f4faf22f80 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0caf01005a5fb704a147ee46c7227b3e changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 335860f4847882dab27e1db49323c60b changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3cfaabbc55e590863a7ecf45872977b3 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 968f86830e42ee358d5e11672847740f changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 79f1ff709eae178b11e9bcfa1e4131b2 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eedb1cdc1fad3f780344bbfed739194f changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4a61d50f100ead183465a200984a5e77 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region edc306f98fb391a3fdc7623063a1cbf8 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b85ee4f1fa33a9cbdcb45c32b6176c88 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fc23d7fa863cb970ac753db9e4e26b26 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region adbccc292c5b7e61d47646b64c419a44 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d17b306f903adc2b73f818b128d5cd67 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 603677e75f480cfd74204fce51bd5359 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2fed3d0dc5a44acae9cbb7359cdf2051 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 796f7fc8fad8d003c466a3f53b960815 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b272925d8b30354bd56e190c0c041139 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8695d5f6c29ed614cc7e97c2ba4c6724 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1229242f433b46943919d6643a95d4a4 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ff2b8d760c6f5acc07476412531886a0 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 360d0b423433ec91f787be3bde665606 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:14:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-27T01:14:39,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-27T01:14:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-27T01:14:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:14:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-27T01:14:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-27T01:14:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-27T01:14:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:14:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:14:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T01:14:39,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:14:39,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:14:39,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T01:14:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-27T01:14:39,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-27T01:14:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-27T01:14:39,320 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:14:39,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-27T01:14:39,367 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=377, OpenFileDescriptor=646, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=6293 2024-11-27T01:14:39,371 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-27T01:14:39,372 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-27T01:14:39,373 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:39,373 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 7d6f26a4c136,32943,1732670073882: Simulated kill ***** 2024-11-27T01:14:39,374 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-27T01:14:39,376 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-27T01:14:39,379 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-27T01:14:39,379 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-27T01:14:39,379 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-27T01:14:39,380 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 308857760 }, "NonHeapMemoryUsage": { "committed": 150798336, "init": 7667712, "max": -1, "used": 148477184 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "7d6f26a4c136" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "7d6f26a4c136" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "7d6f26a4c136" } ] } 2024-11-27T01:14:39,383 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] master.MasterRpcServices(700): 7d6f26a4c136,32943,1732670073882 reported a fatal error: ***** ABORTING region server 7d6f26a4c136,32943,1732670073882: Simulated kill ***** 2024-11-27T01:14:39,385 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,32943,1732670073882' ***** 2024-11-27T01:14:39,385 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-27T01:14:39,385 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:14:39,385 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:14:39,386 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:14:39,386 INFO [RS:2;7d6f26a4c136:32943 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-27T01:14:39,386 INFO [RS:2;7d6f26a4c136:32943 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-27T01:14:39,386 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:14:39,386 INFO [RS:2;7d6f26a4c136:32943 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;7d6f26a4c136:32943. 2024-11-27T01:14:39,387 DEBUG [RS:2;7d6f26a4c136:32943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:14:39,393 DEBUG [RS:2;7d6f26a4c136:32943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:39,398 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:14:39,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741836_1012 (size=32114) 2024-11-27T01:14:39,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741836_1012 (size=32114) 2024-11-27T01:14:39,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741836_1012 (size=32114) 2024-11-27T01:14:39,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741834_1010 (size=2237) 2024-11-27T01:14:39,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741834_1010 (size=2237) 2024-11-27T01:14:39,416 DEBUG [RS:2;7d6f26a4c136:32943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:39,417 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:14:39,417 INFO [RS:2;7d6f26a4c136:32943 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32943 2024-11-27T01:14:39,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741834_1010 (size=2237) 2024-11-27T01:14:39,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:14:39,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,32943,1732670073882 2024-11-27T01:14:39,429 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:14:39,430 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,32943,1732670073882] 2024-11-27T01:14:39,432 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,32943,1732670073882 already deleted, retry=false 2024-11-27T01:14:39,432 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 7d6f26a4c136,32943,1732670073882 on 7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:39,439 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:39,444 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:39,445 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 7d6f26a4c136,32943,1732670073882 (carryingMeta=true) 7d6f26a4c136,32943,1732670073882/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@31aa1c32[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-27T01:14:39,447 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true, isMeta: true 2024-11-27T01:14:39,450 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting 2024-11-27T01:14:39,453 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7d6f26a4c136,32943,1732670073882 WAL count=1, meta=true 2024-11-27T01:14:39,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta}] 2024-11-27T01:14:39,465 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:39,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:39,488 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:14:39,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:14:39,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10156fd0ddf0003, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:14:39,533 INFO [RS:2;7d6f26a4c136:32943 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:14:39,534 INFO [RS:2;7d6f26a4c136:32943 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,32943,1732670073882; zookeeper connection closed. 2024-11-27T01:14:39,536 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6fd510db {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6fd510db 2024-11-27T01:14:39,585 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T01:14:39,585 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T01:14:39,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40647 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-27T01:14:39,690 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, size=31.4 K (32114bytes) 2024-11-27T01:14:39,691 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-27T01:14:39,691 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-27T01:14:39,691 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta 2024-11-27T01:14:39,699 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta after 5ms 2024-11-27T01:14:39,723 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta took 35ms 2024-11-27T01:14:39,738 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-27T01:14:39,748 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta.temp 2024-11-27T01:14:39,752 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta.temp 2024-11-27T01:14:39,755 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta so closing down 2024-11-27T01:14:39,755 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:14:39,769 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:14:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741864_1040 (size=31873) 2024-11-27T01:14:39,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741864_1040 (size=31873) 2024-11-27T01:14:39,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741864_1040 (size=31873) 2024-11-27T01:14:39,802 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta.temp (wrote 57 edits, skipped 0 edits in 15 ms) 2024-11-27T01:14:39,809 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta.temp to hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-27T01:14:39,813 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 82 ms; skipped=1; WAL=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-27T01:14:39,813 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, journal: Splitting hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, size=31.4 K (32114bytes) at 1732670079690Creating recovered edits writer path=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta.temp at 1732670079752 (+62 ms)Finishing writing output for hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta so closing down at 1732670079755 (+3 ms)3 split writer threads finished at 1732670079769 (+14 ms)Closed recovered edits writer path=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta.temp (wrote 57 edits, skipped 0 edits in 15 ms) at 1732670079802 (+33 ms)Rename recovered edits hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta.temp to hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732670079809 (+7 ms)Processed 58 edits across 1 Regions in 82 ms; skipped=1; WAL=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1732670079813 (+4 ms) 2024-11-27T01:14:39,814 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta 2024-11-27T01:14:39,816 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-27T01:14:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-27T01:14:39,834 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta to hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs 2024-11-27T01:14:39,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-27T01:14:39,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 in 366 msec 2024-11-27T01:14:39,841 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:39,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-27T01:14:39,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 in 387 msec 2024-11-27T01:14:39,849 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting dir is empty, no logs to split. 2024-11-27T01:14:39,849 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7d6f26a4c136,32943,1732670073882 WAL count=0, meta=true 2024-11-27T01:14:39,849 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 7d6f26a4c136,32943,1732670073882 WAL splitting is done? wals=0, meta=true 2024-11-27T01:14:39,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T01:14:39,854 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:39,855 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:14:40,006 DEBUG [7d6f26a4c136:33239 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-27T01:14:40,006 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:14:40,006 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:14:40,006 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:14:40,006 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:14:40,006 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:14:40,006 INFO [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:14:40,006 INFO [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:14:40,006 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:14:40,007 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:40,009 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPENING 2024-11-27T01:14:40,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:40,011 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:40,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:40,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:40,013 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:40,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:40,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:40,014 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:40,174 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T01:14:40,174 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:14:40,174 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T01:14:40,177 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C40647%2C1732670073826.meta, suffix=.meta, logDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,40647,1732670073826, archiveDir=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs, maxLogs=32 2024-11-27T01:14:40,197 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,40647,1732670073826/7d6f26a4c136%2C40647%2C1732670073826.meta.1732670080179.meta, exclude list is [], retry=0 2024-11-27T01:14:40,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46391,DS-868d7a4e-f180-4a29-adfe-c9198fcd7493,DISK] 2024-11-27T01:14:40,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40703,DS-ff782df8-ea5f-46cc-9d82-6b5a6f27a76c,DISK] 2024-11-27T01:14:40,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39429,DS-34aa0431-5d12-49f8-a64c-c90b8fe9f878,DISK] 2024-11-27T01:14:40,214 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,40647,1732670073826/7d6f26a4c136%2C40647%2C1732670073826.meta.1732670080179.meta 2024-11-27T01:14:40,214 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42521:42521),(127.0.0.1/127.0.0.1:34009:34009),(127.0.0.1/127.0.0.1:34837:34837)] 2024-11-27T01:14:40,215 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:14:40,216 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T01:14:40,216 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T01:14:40,216 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T01:14:40,220 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T01:14:40,220 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:40,221 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T01:14:40,221 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T01:14:40,240 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T01:14:40,243 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T01:14:40,243 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:40,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:40,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T01:14:40,249 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T01:14:40,249 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:40,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:40,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T01:14:40,257 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T01:14:40,258 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:40,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:40,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T01:14:40,261 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T01:14:40,261 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:40,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:14:40,261 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T01:14:40,263 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740 2024-11-27T01:14:40,266 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740 2024-11-27T01:14:40,267 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-27T01:14:40,304 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-27T01:14:40,308 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-27T01:14:40,480 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/info/9eafd21e2ced40f2ba124c7f4d48c57e is 138, key is test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3./info:regioninfo/1732670078983/Put/seqid=0 2024-11-27T01:14:40,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741866_1042 (size=34924) 2024-11-27T01:14:40,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741866_1042 (size=34924) 2024-11-27T01:14:40,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741866_1042 (size=34924) 2024-11-27T01:14:40,576 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/info/9eafd21e2ced40f2ba124c7f4d48c57e 2024-11-27T01:14:40,671 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/ns/89ad4147d91a42a497dc3254130dd469 is 43, key is default/ns:d/1732670076529/Put/seqid=0 2024-11-27T01:14:40,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741867_1043 (size=5153) 2024-11-27T01:14:40,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741867_1043 (size=5153) 2024-11-27T01:14:40,698 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/ns/89ad4147d91a42a497dc3254130dd469 2024-11-27T01:14:40,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741867_1043 (size=5153) 2024-11-27T01:14:40,739 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/table/18d32e41a2ac408faa23638737695de4 is 40, key is test/table:state/1732670079059/Put/seqid=0 2024-11-27T01:14:40,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741868_1044 (size=5165) 2024-11-27T01:14:40,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741868_1044 (size=5165) 2024-11-27T01:14:40,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741868_1044 (size=5165) 2024-11-27T01:14:40,783 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/table/18d32e41a2ac408faa23638737695de4 2024-11-27T01:14:40,795 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/info/9eafd21e2ced40f2ba124c7f4d48c57e as hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/info/9eafd21e2ced40f2ba124c7f4d48c57e 2024-11-27T01:14:40,807 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/info/9eafd21e2ced40f2ba124c7f4d48c57e, entries=260, sequenceid=60, filesize=34.1 K 2024-11-27T01:14:40,810 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/ns/89ad4147d91a42a497dc3254130dd469 as hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/ns/89ad4147d91a42a497dc3254130dd469 2024-11-27T01:14:40,821 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/ns/89ad4147d91a42a497dc3254130dd469, entries=2, sequenceid=60, filesize=5.0 K 2024-11-27T01:14:40,823 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/table/18d32e41a2ac408faa23638737695de4 as hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/table/18d32e41a2ac408faa23638737695de4 2024-11-27T01:14:40,837 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/table/18d32e41a2ac408faa23638737695de4, entries=2, sequenceid=60, filesize=5.0 K 2024-11-27T01:14:40,844 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 533ms, sequenceid=60, compaction requested=false; wal=null 2024-11-27T01:14:40,846 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T01:14:40,850 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-27T01:14:40,853 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T01:14:40,853 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T01:14:40,855 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:14:40,860 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T01:14:40,869 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-27T01:14:40,871 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59992715, jitterRate=-0.1060388833284378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:14:40,871 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T01:14:40,871 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732670080221Writing region info on filesystem at 1732670080221Initializing all the Stores at 1732670080225 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670080225Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670080236 (+11 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670080237 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670080237Obtaining lock to block concurrent updates at 1732670080308 (+71 ms)Preparing flush snapshotting stores in 1588230740 at 1732670080308Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732670080312 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732670080313 (+1 ms)Flushing 1588230740/info: creating writer at 1732670080323 (+10 ms)Flushing 1588230740/info: appending metadata at 1732670080463 (+140 ms)Flushing 1588230740/info: closing flushed file at 1732670080467 (+4 ms)Flushing 1588230740/ns: creating writer at 1732670080639 (+172 ms)Flushing 1588230740/ns: appending metadata at 1732670080670 (+31 ms)Flushing 1588230740/ns: closing flushed file at 1732670080670Flushing 1588230740/table: creating writer at 1732670080709 (+39 ms)Flushing 1588230740/table: appending metadata at 1732670080738 (+29 ms)Flushing 1588230740/table: closing flushed file at 1732670080738Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@274c8972: reopening flushed file at 1732670080794 (+56 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba4c31e: reopening flushed file at 1732670080809 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@267297dd: reopening flushed file at 1732670080821 (+12 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 533ms, sequenceid=60, compaction requested=false; wal=null at 1732670080844 (+23 ms)Cleaning up temporary data from old regions at 1732670080853 (+9 ms)Running coprocessor post-open hooks at 1732670080871 (+18 ms)Region opened successfully at 1732670080871 2024-11-27T01:14:40,874 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1732670080167 2024-11-27T01:14:40,879 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:40,880 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T01:14:40,880 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T01:14:40,881 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:40,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:40,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:40,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:40,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:40,884 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:40,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:40,885 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:40,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-27T01:14:40,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 in 871 msec 2024-11-27T01:14:40,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-27T01:14:40,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0360 sec 2024-11-27T01:14:40,894 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 7d6f26a4c136,32943,1732670073882 had 9 regions 2024-11-27T01:14:40,895 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true, isMeta: false 2024-11-27T01:14:40,898 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7d6f26a4c136,32943,1732670073882 WAL count=1, meta=false 2024-11-27T01:14:40,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802}] 2024-11-27T01:14:40,904 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:40,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:41,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40647 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-27T01:14:41,082 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802, size=2.2 K (2237bytes) 2024-11-27T01:14:41,082 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802 2024-11-27T01:14:41,082 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802 after 0ms 2024-11-27T01:14:41,086 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802 took 5ms 2024-11-27T01:14:41,092 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 0caf01005a5fb704a147ee46c7227b3e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T01:14:41,094 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 335860f4847882dab27e1db49323c60b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T01:14:41,095 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8af3d7357dfa0f8aa36ba6b01663d6f6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T01:14:41,097 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8ae4c3f2c0fe2fac2f19395fca402eb5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T01:14:41,099 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a05cacf8eaf88a4edb7255a00c86d78c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T01:14:41,100 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8e92cc2f48378cc4a64c15f36bcf6052: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T01:14:41,102 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4b3cb4626ebbddc356f324f4faf22f80: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T01:14:41,104 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for bbecc4f063e3fd6151247c0507c5bef3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T01:14:41,104 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802 so closing down 2024-11-27T01:14:41,104 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:14:41,105 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:14:41,105 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 17 ms; skipped=8; WAL=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-27T01:14:41,105 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802, journal: Splitting hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802, size=2.2 K (2237bytes) at 1732670081082Finishing writing output for hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802 so closing down at 1732670081104 (+22 ms)3 split writer threads finished at 1732670081105 (+1 ms)Processed 8 edits across 0 Regions in 17 ms; skipped=8; WAL=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732670081105 2024-11-27T01:14:41,105 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802 2024-11-27T01:14:41,105 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-27T01:14:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-27T01:14:41,116 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting/7d6f26a4c136%2C32943%2C1732670073882.1732670075802 to hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs 2024-11-27T01:14:41,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-27T01:14:41,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 in 208 msec 2024-11-27T01:14:41,123 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-27T01:14:41,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 in 224 msec 2024-11-27T01:14:41,129 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/WALs/7d6f26a4c136,32943,1732670073882-splitting dir is empty, no logs to split. 2024-11-27T01:14:41,129 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7d6f26a4c136,32943,1732670073882 WAL count=0, meta=false 2024-11-27T01:14:41,129 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 7d6f26a4c136,32943,1732670073882 WAL splitting is done? wals=0, meta=false 2024-11-27T01:14:41,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN}] 2024-11-27T01:14:41,135 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:41,136 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:14:41,140 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:41,142 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:14:41,144 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:41,146 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:14:41,148 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:41,150 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:14:41,151 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:41,152 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:14:41,154 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:41,156 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:14:41,157 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:41,158 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:14:41,159 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:41,160 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:14:41,287 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:14:41,288 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:14:41,288 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:14:41,288 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:14:41,288 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:14:41,288 INFO [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:14:41,288 INFO [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:14:41,288 DEBUG [7d6f26a4c136:33239 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:14:41,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,292 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7d6f26a4c136:32943 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:14:41,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, regionLocation=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=335860f4847882dab27e1db49323c60b, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,294 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1, error=java.net.ConnectException: Call to address=7d6f26a4c136:32943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,294 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, regionLocation=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-27T01:14:41,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 from cache 2024-11-27T01:14:41,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 7d6f26a4c136:32943 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,297 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7d6f26a4c136:32943 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:14:41,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7d6f26a4c136:32943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7d6f26a4c136:32943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7d6f26a4c136:32943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7d6f26a4c136:32943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7d6f26a4c136:32943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7d6f26a4c136:32943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,32943,1732670073882, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7d6f26a4c136:32943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 7d6f26a4c136:32943 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7d6f26a4c136/172.17.0.2:32943 2024-11-27T01:14:41,404 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:14:41,405 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,40647,1732670073826, seqNum=-1] 2024-11-27T01:14:41,405 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:14:41,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34125, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:14:41,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN because future has completed 2024-11-27T01:14:41,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:41,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN because future has completed 2024-11-27T01:14:41,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN because future has completed 2024-11-27T01:14:41,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN because future has completed 2024-11-27T01:14:41,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:41,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN because future has completed 2024-11-27T01:14:41,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN because future has completed 2024-11-27T01:14:41,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN because future has completed 2024-11-27T01:14:41,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN because future has completed 2024-11-27T01:14:41,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:41,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:41,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:41,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:41,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,40647,1732670073826}] 2024-11-27T01:14:41,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e92cc2f48378cc4a64c15f36bcf6052, server=7d6f26a4c136,39915,1732670073690}] 2024-11-27T01:14:41,571 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:41,572 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => a05cacf8eaf88a4edb7255a00c86d78c, NAME => 'test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-27T01:14:41,572 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,572 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:41,572 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,572 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,578 INFO [StoreOpener-a05cacf8eaf88a4edb7255a00c86d78c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,579 INFO [StoreOpener-a05cacf8eaf88a4edb7255a00c86d78c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a05cacf8eaf88a4edb7255a00c86d78c columnFamilyName family 2024-11-27T01:14:41,579 DEBUG [StoreOpener-a05cacf8eaf88a4edb7255a00c86d78c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:41,580 INFO [StoreOpener-a05cacf8eaf88a4edb7255a00c86d78c-1 {}] regionserver.HStore(327): Store=a05cacf8eaf88a4edb7255a00c86d78c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:41,580 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,581 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,583 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,583 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,583 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,586 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,587 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:41,587 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ae4c3f2c0fe2fac2f19395fca402eb5, NAME => 'test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-27T01:14:41,587 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,587 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:41,587 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,587 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,590 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened a05cacf8eaf88a4edb7255a00c86d78c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69355182, jitterRate=0.03347274661064148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:41,590 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:41,590 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for a05cacf8eaf88a4edb7255a00c86d78c: Running coprocessor pre-open hook at 1732670081573Writing region info on filesystem at 1732670081573Initializing all the Stores at 1732670081578 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670081578Cleaning up temporary data from old regions at 1732670081583 (+5 ms)Running coprocessor post-open hooks at 1732670081590 (+7 ms)Region opened successfully at 1732670081590 2024-11-27T01:14:41,590 INFO [StoreOpener-8ae4c3f2c0fe2fac2f19395fca402eb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,592 INFO [StoreOpener-8ae4c3f2c0fe2fac2f19395fca402eb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ae4c3f2c0fe2fac2f19395fca402eb5 columnFamilyName family 2024-11-27T01:14:41,592 DEBUG [StoreOpener-8ae4c3f2c0fe2fac2f19395fca402eb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:41,593 INFO [StoreOpener-8ae4c3f2c0fe2fac2f19395fca402eb5-1 {}] regionserver.HStore(327): Store=8ae4c3f2c0fe2fac2f19395fca402eb5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:41,593 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,594 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,595 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,595 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,595 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,596 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c., pid=73, masterSystemTime=1732670081566 2024-11-27T01:14:41,598 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,599 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:41,599 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:41,600 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:41,600 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => bbecc4f063e3fd6151247c0507c5bef3, NAME => 'test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-27T01:14:41,600 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,600 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:41,600 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,600 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,601 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 8ae4c3f2c0fe2fac2f19395fca402eb5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60631243, jitterRate=-0.09652407467365265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:41,601 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:41,601 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 8ae4c3f2c0fe2fac2f19395fca402eb5: Running coprocessor pre-open hook at 1732670081588Writing region info on filesystem at 1732670081588Initializing all the Stores at 1732670081589 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670081590 (+1 ms)Cleaning up temporary data from old regions at 1732670081595 (+5 ms)Running coprocessor post-open hooks at 1732670081601 (+6 ms)Region opened successfully at 1732670081601 2024-11-27T01:14:41,603 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-27T01:14:41,603 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,604 INFO [StoreOpener-bbecc4f063e3fd6151247c0507c5bef3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,604 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5., pid=74, masterSystemTime=1732670081582 2024-11-27T01:14:41,603 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,605 INFO [StoreOpener-bbecc4f063e3fd6151247c0507c5bef3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbecc4f063e3fd6151247c0507c5bef3 columnFamilyName family 2024-11-27T01:14:41,605 DEBUG [StoreOpener-bbecc4f063e3fd6151247c0507c5bef3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:41,606 INFO [StoreOpener-bbecc4f063e3fd6151247c0507c5bef3-1 {}] regionserver.HStore(327): Store=bbecc4f063e3fd6151247c0507c5bef3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:41,606 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,607 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:41,607 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:41,607 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:41,607 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b3cb4626ebbddc356f324f4faf22f80, NAME => 'test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-27T01:14:41,607 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,608 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,608 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:41,608 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,608 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,610 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,611 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,611 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,614 INFO [StoreOpener-4b3cb4626ebbddc356f324f4faf22f80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,614 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,615 INFO [StoreOpener-4b3cb4626ebbddc356f324f4faf22f80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b3cb4626ebbddc356f324f4faf22f80 columnFamilyName family 2024-11-27T01:14:41,615 DEBUG [StoreOpener-4b3cb4626ebbddc356f324f4faf22f80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:41,616 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened bbecc4f063e3fd6151247c0507c5bef3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66549942, jitterRate=-0.00832858681678772}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:41,616 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:41,616 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for bbecc4f063e3fd6151247c0507c5bef3: Running coprocessor pre-open hook at 1732670081600Writing region info on filesystem at 1732670081600Initializing all the Stores at 1732670081602 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670081602Cleaning up temporary data from old regions at 1732670081611 (+9 ms)Running coprocessor post-open hooks at 1732670081616 (+5 ms)Region opened successfully at 1732670081616 2024-11-27T01:14:41,617 INFO [StoreOpener-4b3cb4626ebbddc356f324f4faf22f80-1 {}] regionserver.HStore(327): Store=4b3cb4626ebbddc356f324f4faf22f80/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:41,617 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,616 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-27T01:14:41,618 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3., pid=77, masterSystemTime=1732670081566 2024-11-27T01:14:41,618 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,619 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:41,619 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e92cc2f48378cc4a64c15f36bcf6052, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,622 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e92cc2f48378cc4a64c15f36bcf6052, server=7d6f26a4c136,39915,1732670073690 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,622 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,622 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:41,622 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:41,623 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:41,623 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,623 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,623 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 0caf01005a5fb704a147ee46c7227b3e, NAME => 'test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-27T01:14:41,623 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,623 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:41,623 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,623 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,625 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,627 INFO [StoreOpener-0caf01005a5fb704a147ee46c7227b3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,627 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 4b3cb4626ebbddc356f324f4faf22f80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64201074, jitterRate=-0.043329447507858276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:41,627 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:41,627 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 4b3cb4626ebbddc356f324f4faf22f80: Running coprocessor pre-open hook at 1732670081608Writing region info on filesystem at 1732670081608Initializing all the Stores at 1732670081613 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670081613Cleaning up temporary data from old regions at 1732670081623 (+10 ms)Running coprocessor post-open hooks at 1732670081627 (+4 ms)Region opened successfully at 1732670081627 2024-11-27T01:14:41,628 INFO [StoreOpener-0caf01005a5fb704a147ee46c7227b3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0caf01005a5fb704a147ee46c7227b3e columnFamilyName family 2024-11-27T01:14:41,628 DEBUG [StoreOpener-0caf01005a5fb704a147ee46c7227b3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:41,629 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80., pid=75, masterSystemTime=1732670081582 2024-11-27T01:14:41,629 INFO [StoreOpener-0caf01005a5fb704a147ee46c7227b3e-1 {}] regionserver.HStore(327): Store=0caf01005a5fb704a147ee46c7227b3e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:41,629 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,630 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,633 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,633 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:41,633 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,633 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:41,633 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,634 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:41,634 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 335860f4847882dab27e1db49323c60b, NAME => 'test,xxx,1732670076899.335860f4847882dab27e1db49323c60b.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-27T01:14:41,635 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,635 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,xxx,1732670076899.335860f4847882dab27e1db49323c60b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:41,635 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,635 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,637 INFO [StoreOpener-335860f4847882dab27e1db49323c60b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,638 INFO [StoreOpener-335860f4847882dab27e1db49323c60b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335860f4847882dab27e1db49323c60b columnFamilyName family 2024-11-27T01:14:41,638 DEBUG [StoreOpener-335860f4847882dab27e1db49323c60b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:41,639 INFO [StoreOpener-335860f4847882dab27e1db49323c60b-1 {}] regionserver.HStore(327): Store=335860f4847882dab27e1db49323c60b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:41,639 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,639 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,640 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,640 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 0caf01005a5fb704a147ee46c7227b3e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63321833, jitterRate=-0.056431159377098083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:41,641 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:41,641 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 0caf01005a5fb704a147ee46c7227b3e: Running coprocessor pre-open hook at 1732670081623Writing region info on filesystem at 1732670081623Initializing all the Stores at 1732670081626 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670081626Cleaning up temporary data from old regions at 1732670081634 (+8 ms)Running coprocessor post-open hooks at 1732670081641 (+7 ms)Region opened successfully at 1732670081641 2024-11-27T01:14:41,641 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,642 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e., pid=72, masterSystemTime=1732670081566 2024-11-27T01:14:41,643 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,643 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,646 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,646 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:41,646 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:41,647 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:41,647 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 335860f4847882dab27e1db49323c60b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64211626, jitterRate=-0.043172210454940796}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:41,647 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:41,647 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 8af3d7357dfa0f8aa36ba6b01663d6f6, NAME => 'test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-27T01:14:41,647 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 335860f4847882dab27e1db49323c60b: Running coprocessor pre-open hook at 1732670081635Writing region info on filesystem at 1732670081635Initializing all the Stores at 1732670081637 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670081637Cleaning up temporary data from old regions at 1732670081643 (+6 ms)Running coprocessor post-open hooks at 1732670081647 (+4 ms)Region opened successfully at 1732670081647 2024-11-27T01:14:41,648 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,648 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:41,648 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,648 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,653 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732670076899.335860f4847882dab27e1db49323c60b., pid=76, masterSystemTime=1732670081582 2024-11-27T01:14:41,653 INFO [StoreOpener-8af3d7357dfa0f8aa36ba6b01663d6f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,656 INFO [StoreOpener-8af3d7357dfa0f8aa36ba6b01663d6f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8af3d7357dfa0f8aa36ba6b01663d6f6 columnFamilyName family 2024-11-27T01:14:41,656 DEBUG [StoreOpener-8af3d7357dfa0f8aa36ba6b01663d6f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:41,657 INFO [StoreOpener-8af3d7357dfa0f8aa36ba6b01663d6f6-1 {}] regionserver.HStore(327): Store=8af3d7357dfa0f8aa36ba6b01663d6f6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:41,657 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,657 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:41,657 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:41,657 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:41,658 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,658 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e92cc2f48378cc4a64c15f36bcf6052, NAME => 'test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-27T01:14:41,658 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,658 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:14:41,659 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,659 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,660 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,661 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,661 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,661 INFO [StoreOpener-8e92cc2f48378cc4a64c15f36bcf6052-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,663 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,663 INFO [StoreOpener-8e92cc2f48378cc4a64c15f36bcf6052-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e92cc2f48378cc4a64c15f36bcf6052 columnFamilyName family 2024-11-27T01:14:41,663 DEBUG [StoreOpener-8e92cc2f48378cc4a64c15f36bcf6052-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:14:41,664 INFO [StoreOpener-8e92cc2f48378cc4a64c15f36bcf6052-1 {}] regionserver.HStore(327): Store=8e92cc2f48378cc4a64c15f36bcf6052/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:14:41,664 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,664 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 8af3d7357dfa0f8aa36ba6b01663d6f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66250946, jitterRate=-0.012783974409103394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:41,664 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:41,664 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 8af3d7357dfa0f8aa36ba6b01663d6f6: Running coprocessor pre-open hook at 1732670081648Writing region info on filesystem at 1732670081648Initializing all the Stores at 1732670081653 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670081653Cleaning up temporary data from old regions at 1732670081661 (+8 ms)Running coprocessor post-open hooks at 1732670081664 (+3 ms)Region opened successfully at 1732670081664 2024-11-27T01:14:41,666 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,667 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6., pid=78, masterSystemTime=1732670081566 2024-11-27T01:14:41,670 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:41,670 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:41,670 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,671 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,671 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,673 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,675 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 8e92cc2f48378cc4a64c15f36bcf6052; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69140310, jitterRate=0.030270904302597046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:14:41,675 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:41,675 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 8e92cc2f48378cc4a64c15f36bcf6052: Running coprocessor pre-open hook at 1732670081659Writing region info on filesystem at 1732670081659Initializing all the Stores at 1732670081661 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670081661Cleaning up temporary data from old regions at 1732670081671 (+10 ms)Running coprocessor post-open hooks at 1732670081675 (+4 ms)Region opened successfully at 1732670081675 2024-11-27T01:14:41,677 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052., pid=79, masterSystemTime=1732670081582 2024-11-27T01:14:41,681 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:41,681 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:41,695 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:41,695 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,695 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:41,695 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:41,696 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:41,696 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:41,696 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:41,696 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:41,697 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:41,703 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:41,703 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,703 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,703 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,703 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,703 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,703 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,703 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:41,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:41,705 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:41,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e92cc2f48378cc4a64c15f36bcf6052, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e92cc2f48378cc4a64c15f36bcf6052, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,713 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-27T01:14:41,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:41,715 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:41,715 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:41,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,726 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:41,727 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:41,727 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:41,727 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:41,727 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:41,727 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:41,727 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:41,727 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN to state=OFFLINE, location=null, table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6 to restore RIT 2024-11-27T01:14:41,728 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670111728 2024-11-27T01:14:41,728 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:41,728 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,734 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:41,739 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:41,739 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,739 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,40647,1732670073826 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,776 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,777 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,777 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,777 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,778 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,778 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,778 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,781 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:41,781 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670141781 2024-11-27T01:14:41,781 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670201781 2024-11-27T01:14:41,784 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 52 msec 2024-11-27T01:14:41,784 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:41,784 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,784 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:41,785 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:41,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:41,785 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:41,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:41,786 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:41,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:41,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:41,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:41,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:41,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:41,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:41,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,798 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,798 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,798 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,798 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,798 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8af3d7357dfa0f8aa36ba6b01663d6f6, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-27T01:14:41,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:41,800 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:41,800 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:41,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,814 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:41,814 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:41,814 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:41,814 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:41,814 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:41,814 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:41,814 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:41,814 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN to state=OFFLINE, location=null, table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6 to restore RIT 2024-11-27T01:14:41,828 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670111828 2024-11-27T01:14:41,828 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:41,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,829 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,844 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:41,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,846 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:41,846 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,846 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,40647,1732670073826 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,849 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:41,849 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670141849 2024-11-27T01:14:41,849 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670201849 2024-11-27T01:14:41,849 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 20 msec 2024-11-27T01:14:41,849 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:41,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,849 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:41,849 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:41,852 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:41,853 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:41,853 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:41,855 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:41,858 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-27T01:14:41,859 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,873 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,883 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,884 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:41,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:41,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:41,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:41,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:41,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:41,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:41,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:41,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,892 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,893 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,893 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:41,893 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,894 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:14:41,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,894 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbecc4f063e3fd6151247c0507c5bef3, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,894 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 36 msec 2024-11-27T01:14:41,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:41,897 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:41,897 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:41,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,899 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,899 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:41,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:41,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:41,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:41,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:41,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:41,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:41,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN to state=OFFLINE, location=null, table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6 to restore RIT 2024-11-27T01:14:41,920 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670111920 2024-11-27T01:14:41,920 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:41,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,929 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,931 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:41,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,932 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:41,933 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,937 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:41,937 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670141937 2024-11-27T01:14:41,937 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670201937 2024-11-27T01:14:41,938 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 17 msec 2024-11-27T01:14:41,938 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:41,938 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,938 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:41,938 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:41,938 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:41,938 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:41,938 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:41,939 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:41,940 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T01:14:41,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:41,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:41,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:41,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:41,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335860f4847882dab27e1db49323c60b, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:41,957 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-27T01:14:41,958 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:41,958 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:41,959 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:41,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:41,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:41,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:41,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:41,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:41,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:41,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:41,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:41,963 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:41,963 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN to state=OFFLINE, location=null, table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6 to restore RIT 2024-11-27T01:14:41,977 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670111977 2024-11-27T01:14:41,977 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:41,980 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,982 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:41,984 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:41,984 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:41,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,984 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:41,993 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:41,993 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:41,993 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670141993 2024-11-27T01:14:41,993 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670201993 2024-11-27T01:14:41,993 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-27T01:14:41,993 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:41,993 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:41,993 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:41,993 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:41,994 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:41,994 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:41,994 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:41,994 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:41,995 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,000 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,004 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,007 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,007 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,007 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,008 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,008 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,008 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,008 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b3cb4626ebbddc356f324f4faf22f80, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,010 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-27T01:14:42,010 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-27T01:14:42,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,011 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,011 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,014 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,014 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,015 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,015 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,015 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN to state=OFFLINE, location=null, table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6 to restore RIT 2024-11-27T01:14:42,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,016 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,021 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112021 2024-11-27T01:14:42,021 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,021 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,023 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,026 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,026 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:42,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,026 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,034 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142034 2024-11-27T01:14:42,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202034 2024-11-27T01:14:42,034 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-27T01:14:42,034 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,034 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,034 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,034 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,035 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,035 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,035 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,035 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,036 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,040 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,045 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,045 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,045 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,045 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,046 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,046 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,046 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,046 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,046 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,046 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,046 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,046 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,046 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,046 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,046 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,046 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ae4c3f2c0fe2fac2f19395fca402eb5, server=7d6f26a4c136,39915,1732670073690 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,046 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-27T01:14:42,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,047 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,047 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,051 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,051 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,053 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,053 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,053 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,053 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:42,053 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:42,053 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:42,053 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:42,054 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN to state=OFFLINE, location=null, table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6 to restore RIT 2024-11-27T01:14:42,059 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112059 2024-11-27T01:14:42,059 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,059 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,063 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,066 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,066 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,066 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,074 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,074 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142074 2024-11-27T01:14:42,074 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202074 2024-11-27T01:14:42,074 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-27T01:14:42,074 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,074 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,074 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,074 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,074 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,074 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,074 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,075 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,076 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,079 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,080 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,080 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,080 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,081 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,081 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,085 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,087 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,087 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,088 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,088 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,088 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,088 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,088 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,088 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,088 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,088 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,088 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,088 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,088 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,088 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,088 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,088 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,088 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-27T01:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,089 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,089 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,091 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,091 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,091 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,093 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,093 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,093 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,093 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:42,093 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:42,093 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:42,093 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:42,093 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN to state=OFFLINE, location=null, table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6 to restore RIT 2024-11-27T01:14:42,096 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112096 2024-11-27T01:14:42,096 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,096 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,098 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,099 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,099 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,099 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,110 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,110 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142110 2024-11-27T01:14:42,110 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202110 2024-11-27T01:14:42,110 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-27T01:14:42,111 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,111 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,111 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,111 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,111 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,111 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,112 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:42,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,120 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,120 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0caf01005a5fb704a147ee46c7227b3e, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,120 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-27T01:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,127 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,128 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN to state=OFFLINE, location=null, table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6 to restore RIT 2024-11-27T01:14:42,134 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112134 2024-11-27T01:14:42,135 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,135 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,136 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,137 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,137 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,138 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,151 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142151 2024-11-27T01:14:42,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202151 2024-11-27T01:14:42,151 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-27T01:14:42,151 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,151 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,151 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,151 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,152 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,152 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,160 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,160 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,160 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,160 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,160 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,161 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,161 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,161 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,161 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,161 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,161 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,161 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,161 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN 2024-11-27T01:14:42,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-27T01:14:42,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,166 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,166 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,168 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,175 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,175 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,175 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,175 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:42,175 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:42,176 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:42,176 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:42,176 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6, ASSIGN to state=OFFLINE, location=null, table=test, region=8af3d7357dfa0f8aa36ba6b01663d6f6 to restore RIT 2024-11-27T01:14:42,177 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112177 2024-11-27T01:14:42,177 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,177 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,181 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,182 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,182 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,183 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,193 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,193 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142193 2024-11-27T01:14:42,193 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202193 2024-11-27T01:14:42,193 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-27T01:14:42,194 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,194 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,194 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,200 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,200 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,201 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,201 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,203 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-27T01:14:42,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,209 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,211 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,211 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,211 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,213 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN 2024-11-27T01:14:42,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-27T01:14:42,213 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,227 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,227 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:42,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:42,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:42,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a05cacf8eaf88a4edb7255a00c86d78c, ASSIGN to state=OFFLINE, location=null, table=test, region=a05cacf8eaf88a4edb7255a00c86d78c to restore RIT 2024-11-27T01:14:42,248 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112248 2024-11-27T01:14:42,248 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,248 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,250 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,252 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,252 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,252 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,259 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,259 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142259 2024-11-27T01:14:42,259 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202259 2024-11-27T01:14:42,259 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-27T01:14:42,259 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,259 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,260 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,260 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,260 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,260 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,260 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,261 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,266 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,266 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,269 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,269 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,269 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,269 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,269 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,269 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,269 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,269 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,269 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,269 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,269 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN 2024-11-27T01:14:42,269 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-27T01:14:42,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,270 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,270 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5, ASSIGN to state=OFFLINE, location=null, table=test, region=8ae4c3f2c0fe2fac2f19395fca402eb5 to restore RIT 2024-11-27T01:14:42,280 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112280 2024-11-27T01:14:42,280 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,280 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,282 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,283 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,283 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,283 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,291 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,291 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142291 2024-11-27T01:14:42,291 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202291 2024-11-27T01:14:42,291 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-27T01:14:42,291 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,291 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,291 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,291 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,291 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,291 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,291 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,292 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,293 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,300 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,300 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,300 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,300 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,300 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,300 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN 2024-11-27T01:14:42,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-27T01:14:42,301 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,301 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,302 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:42,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8e92cc2f48378cc4a64c15f36bcf6052, ASSIGN to state=OFFLINE, location=null, table=test, region=8e92cc2f48378cc4a64c15f36bcf6052 to restore RIT 2024-11-27T01:14:42,324 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112324 2024-11-27T01:14:42,324 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,324 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,326 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,328 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,328 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,328 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,334 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,334 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142334 2024-11-27T01:14:42,334 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202334 2024-11-27T01:14:42,334 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-27T01:14:42,334 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,334 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,334 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,334 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,335 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,335 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,335 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,336 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN 2024-11-27T01:14:42,341 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T01:14:42,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,349 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,350 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,353 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,354 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,354 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,354 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,354 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0caf01005a5fb704a147ee46c7227b3e, ASSIGN to state=OFFLINE, location=null, table=test, region=0caf01005a5fb704a147ee46c7227b3e to restore RIT 2024-11-27T01:14:42,354 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,361 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112361 2024-11-27T01:14:42,361 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,361 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,364 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,365 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,365 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,365 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,376 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,376 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142376 2024-11-27T01:14:42,376 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202376 2024-11-27T01:14:42,376 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-27T01:14:42,376 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,376 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,376 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,376 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,379 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,379 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,379 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,379 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,380 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:42,382 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,382 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,382 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,382 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,382 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,382 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,382 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,382 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,383 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,383 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,383 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,383 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,385 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,386 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,386 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,386 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,386 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,386 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,386 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN 2024-11-27T01:14:42,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T01:14:42,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,387 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,387 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,392 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,392 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,392 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbecc4f063e3fd6151247c0507c5bef3, ASSIGN to state=OFFLINE, location=null, table=test, region=bbecc4f063e3fd6151247c0507c5bef3 to restore RIT 2024-11-27T01:14:42,394 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112394 2024-11-27T01:14:42,394 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,402 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,402 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142402 2024-11-27T01:14:42,402 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202402 2024-11-27T01:14:42,402 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-27T01:14:42,394 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,405 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,406 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,406 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,407 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,652 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,653 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,653 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,653 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,653 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,653 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,653 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,654 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:42,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,659 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,659 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,659 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN 2024-11-27T01:14:42,659 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T01:14:42,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,660 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,661 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,668 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,668 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,668 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,669 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,669 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b3cb4626ebbddc356f324f4faf22f80, ASSIGN to state=OFFLINE, location=null, table=test, region=4b3cb4626ebbddc356f324f4faf22f80 to restore RIT 2024-11-27T01:14:42,687 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112687 2024-11-27T01:14:42,687 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,687 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,689 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,696 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,696 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,696 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,700 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,700 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142700 2024-11-27T01:14:42,700 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202700 2024-11-27T01:14:42,700 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-27T01:14:42,700 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:42,700 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,700 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:42,702 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:42,703 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:42,703 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:42,703 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:42,703 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:42,706 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-27T01:14:42,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:42,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:42,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:42,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:42,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:42,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:42,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:42,716 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:42,716 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN 2024-11-27T01:14:42,716 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-27T01:14:42,725 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,728 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:42,728 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:42,729 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:42,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:42,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:42,734 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=335860f4847882dab27e1db49323c60b, ASSIGN to state=OFFLINE, location=null, table=test, region=335860f4847882dab27e1db49323c60b to restore RIT 2024-11-27T01:14:42,742 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670112742 2024-11-27T01:14:42,742 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:42,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:42,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:42,751 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:42,751 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670142751 2024-11-27T01:14:42,751 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670202751 2024-11-27T01:14:42,751 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-27T01:14:42,747 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:42,752 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:42,754 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:42,754 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:42,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:42,754 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,001 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:43,001 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,002 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:43,002 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:43,002 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:43,002 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:43,002 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:43,002 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:43,003 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:43,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:43,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:43,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:43,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:43,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:43,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:43,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:43,005 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:43,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,008 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T01:14:43,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:43,010 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:43,010 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:43,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,015 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,015 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,016 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,022 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670113022 2024-11-27T01:14:43,023 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:43,024 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,025 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:43,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:43,027 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,031 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:43,031 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670143031 2024-11-27T01:14:43,031 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670203031 2024-11-27T01:14:43,031 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-27T01:14:43,031 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:43,031 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,031 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:43,031 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:43,032 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:43,032 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:43,032 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:43,032 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:43,033 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:43,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:43,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:43,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:43,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:43,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:43,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:43,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:43,037 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:43,037 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.1732670075802, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,038 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T01:14:43,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:43,042 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:43,042 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:43,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,065 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670113065 2024-11-27T01:14:43,065 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:43,067 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,069 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:43,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,077 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:43,077 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670143077 2024-11-27T01:14:43,077 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670203077 2024-11-27T01:14:43,077 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-27T01:14:43,078 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:43,078 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,078 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,264 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T01:14:43,265 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T01:14:43,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-27T01:14:43,267 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-27T01:14:43,268 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T01:14:43,268 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T01:14:43,268 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T01:14:43,269 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T01:14:43,327 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:43,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,327 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:43,327 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:43,328 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:43,328 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:43,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:43,328 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:43,329 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:43,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:43,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:43,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:43,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:43,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T01:14:43,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:43,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:43,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:43,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:43,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:43,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,333 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-27T01:14:43,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:43,334 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:43,334 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:43,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670113341 2024-11-27T01:14:43,341 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:43,342 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:43,346 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:43,346 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:43,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,347 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,350 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:43,350 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670143350 2024-11-27T01:14:43,350 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670203350 2024-11-27T01:14:43,350 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-27T01:14:43,350 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:43,350 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,350 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:43,350 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:43,350 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:43,350 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:43,350 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:43,351 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:43,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:43,353 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:43,353 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:43,353 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:43,353 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T01:14:43,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:43,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:43,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:43,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:14:43,356 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:43,356 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,356 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,356 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,356 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T01:14:43,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:43,357 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:43,357 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:43,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,368 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,369 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,372 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670113372 2024-11-27T01:14:43,372 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:43,374 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,375 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:43,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,380 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:43,380 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670143380 2024-11-27T01:14:43,380 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670203380 2024-11-27T01:14:43,380 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-27T01:14:43,381 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:43,381 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,381 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,630 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:43,630 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,630 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:43,630 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:43,631 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:43,631 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:43,631 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:43,631 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:43,632 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:43,633 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:43,634 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:43,634 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T01:14:43,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:43,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:43,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:43,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:43,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,636 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-27T01:14:43,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:43,643 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:43,643 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:43,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,655 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670113655 2024-11-27T01:14:43,655 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:43,655 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,656 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:43,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,661 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:43,661 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,661 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,663 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:43,663 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670143663 2024-11-27T01:14:43,663 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670203663 2024-11-27T01:14:43,663 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-27T01:14:43,663 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:43,663 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,663 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:43,663 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:43,664 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:43,664 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:43,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:43,665 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:43,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T01:14:43,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:43,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T01:14:43,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:43,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:43,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:43,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:43,669 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:43,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C32943%2C1732670073882.meta.1732670076325.meta, worker=7d6f26a4c136,40647,1732670073826 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T01:14:43,669 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-27T01:14:43,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:43,670 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:43,670 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:43,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,684 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670113684 2024-11-27T01:14:43,684 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:43,684 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,690 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:43,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,694 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:43,694 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670143694 2024-11-27T01:14:43,694 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670203694 2024-11-27T01:14:43,694 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-27T01:14:43,695 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T01:14:43,695 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:43,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,695 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,945 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T01:14:43,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:43,945 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:43,945 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:43,945 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T01:14:43,945 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T01:14:43,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T01:14:43,946 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:14:43,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T01:14:43,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T01:14:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:14:43,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T01:14:43,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true 2024-11-27T01:14:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T01:14:43,950 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-27T01:14:43,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=61 2024-11-27T01:14:43,955 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7d6f26a4c136,40647,1732670073826, table=hbase:meta, region=1588230740 2024-11-27T01:14:43,955 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,40647,1732670073826, state=OPEN 2024-11-27T01:14:43,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:14:43,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:14:43,970 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670113970 2024-11-27T01:14:43,970 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:14:43,971 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:14:43,973 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T01:14:43,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cfaabbc55e590863a7ecf45872977b3, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbecc4f063e3fd6151247c0507c5bef3, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d17b306f903adc2b73f818b128d5cd67, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=968f86830e42ee358d5e11672847740f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603677e75f480cfd74204fce51bd5359, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure a05cacf8eaf88a4edb7255a00c86d78c, server=7d6f26a4c136,40647,1732670073826:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7d6f26a4c136,32943,1732670073882, splitWal=true, meta=true exec-time=4.5410 sec 2024-11-27T01:14:43,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e92cc2f48378cc4a64c15f36bcf6052, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f1ff709eae178b11e9bcfa1e4131b2, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a05cacf8eaf88a4edb7255a00c86d78c, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fed3d0dc5a44acae9cbb7359cdf2051, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eedb1cdc1fad3f780344bbfed739194f, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=796f7fc8fad8d003c466a3f53b960815, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ae4c3f2c0fe2fac2f19395fca402eb5, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a61d50f100ead183465a200984a5e77, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8af3d7357dfa0f8aa36ba6b01663d6f6, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b272925d8b30354bd56e190c0c041139, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8695d5f6c29ed614cc7e97c2ba4c6724, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc306f98fb391a3fdc7623063a1cbf8, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b3cb4626ebbddc356f324f4faf22f80, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b85ee4f1fa33a9cbdcb45c32b6176c88, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1229242f433b46943919d6643a95d4a4, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0caf01005a5fb704a147ee46c7227b3e, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc23d7fa863cb970ac753db9e4e26b26, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff2b8d760c6f5acc07476412531886a0, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335860f4847882dab27e1db49323c60b, regionState=OPENING, lastHost=7d6f26a4c136,32943,1732670073882, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adbccc292c5b7e61d47646b64c419a44, regionState=OPEN, lastHost=7d6f26a4c136,39915,1732670073690, regionLocation=7d6f26a4c136,39915,1732670073690, openSeqNum=2 2024-11-27T01:14:43,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=360d0b423433ec91f787be3bde665606, regionState=OPEN, lastHost=7d6f26a4c136,40647,1732670073826, regionLocation=7d6f26a4c136,40647,1732670073826, openSeqNum=2 2024-11-27T01:14:43,981 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T01:14:43,981 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670143981 2024-11-27T01:14:43,981 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670203981 2024-11-27T01:14:43,981 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-27T01:14:44,038 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=345 (was 377), OpenFileDescriptor=624 (was 646), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 448) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6241 (was 6293) 2024-11-27T01:14:44,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T01:14:44,039 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:14:44,039 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:14:44,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:44,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:44,040 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:14:44,040 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T01:14:44,040 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1872569587, stopped=false 2024-11-27T01:14:44,041 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7d6f26a4c136,33239,1732670072670 2024-11-27T01:14:44,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:14:44,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:14:44,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:44,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:44,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:14:44,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:44,044 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:44,044 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:14:44,044 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:14:44,044 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:14:44,045 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:14:44,045 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:14:44,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:44,046 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,39915,1732670073690' ***** 2024-11-27T01:14:44,046 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T01:14:44,047 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,40647,1732670073826' ***** 2024-11-27T01:14:44,047 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T01:14:44,047 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:14:44,047 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:14:44,047 INFO [RS:0;7d6f26a4c136:39915 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T01:14:44,047 INFO [RS:0;7d6f26a4c136:39915 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T01:14:44,047 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for 3cfaabbc55e590863a7ecf45872977b3 2024-11-27T01:14:44,048 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:14:44,048 INFO [RS:1;7d6f26a4c136:40647 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T01:14:44,048 INFO [RS:1;7d6f26a4c136:40647 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T01:14:44,048 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for bbecc4f063e3fd6151247c0507c5bef3 2024-11-27T01:14:44,048 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:14:44,050 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:14:44,050 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:14:44,050 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for edc306f98fb391a3fdc7623063a1cbf8 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for b85ee4f1fa33a9cbdcb45c32b6176c88 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for 4a61d50f100ead183465a200984a5e77 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ae4c3f2c0fe2fac2f19395fca402eb5 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for adbccc292c5b7e61d47646b64c419a44 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for 79f1ff709eae178b11e9bcfa1e4131b2 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for 968f86830e42ee358d5e11672847740f 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for 4b3cb4626ebbddc356f324f4faf22f80 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for 8e92cc2f48378cc4a64c15f36bcf6052 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(3091): Received CLOSE for 335860f4847882dab27e1db49323c60b 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(959): stopping server 7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:14:44,051 INFO [RS:0;7d6f26a4c136:39915 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7d6f26a4c136:39915. 2024-11-27T01:14:44,051 DEBUG [RS:0;7d6f26a4c136:39915 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:14:44,051 DEBUG [RS:0;7d6f26a4c136:39915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:44,052 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-27T01:14:44,052 DEBUG [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(1351): Waiting on 335860f4847882dab27e1db49323c60b, 3cfaabbc55e590863a7ecf45872977b3, 4a61d50f100ead183465a200984a5e77, 4b3cb4626ebbddc356f324f4faf22f80, 79f1ff709eae178b11e9bcfa1e4131b2, 8ae4c3f2c0fe2fac2f19395fca402eb5, 8e92cc2f48378cc4a64c15f36bcf6052, 968f86830e42ee358d5e11672847740f, adbccc292c5b7e61d47646b64c419a44, b85ee4f1fa33a9cbdcb45c32b6176c88, edc306f98fb391a3fdc7623063a1cbf8, eedb1cdc1fad3f780344bbfed739194f, fc23d7fa863cb970ac753db9e4e26b26 2024-11-27T01:14:44,060 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3cfaabbc55e590863a7ecf45872977b3, disabling compactions & flushes 2024-11-27T01:14:44,060 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:44,060 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:44,060 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. after waiting 0 ms 2024-11-27T01:14:44,060 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for 0caf01005a5fb704a147ee46c7227b3e 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for 2fed3d0dc5a44acae9cbb7359cdf2051 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for a05cacf8eaf88a4edb7255a00c86d78c 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for 8695d5f6c29ed614cc7e97c2ba4c6724 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for 8af3d7357dfa0f8aa36ba6b01663d6f6 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for b272925d8b30354bd56e190c0c041139 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for 360d0b423433ec91f787be3bde665606 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for 796f7fc8fad8d003c466a3f53b960815 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for 1229242f433b46943919d6643a95d4a4 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for 603677e75f480cfd74204fce51bd5359 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(959): stopping server 7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:44,063 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:14:44,064 INFO [RS:1;7d6f26a4c136:40647 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7d6f26a4c136:40647. 2024-11-27T01:14:44,064 DEBUG [RS:1;7d6f26a4c136:40647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:14:44,064 DEBUG [RS:1;7d6f26a4c136:40647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:44,064 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T01:14:44,064 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T01:14:44,064 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T01:14:44,064 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T01:14:44,065 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bbecc4f063e3fd6151247c0507c5bef3, disabling compactions & flushes 2024-11-27T01:14:44,065 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:44,065 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:44,065 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. after waiting 0 ms 2024-11-27T01:14:44,065 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:44,070 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-27T01:14:44,070 DEBUG [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(1351): Waiting on 0caf01005a5fb704a147ee46c7227b3e, 1229242f433b46943919d6643a95d4a4, 1588230740, 2fed3d0dc5a44acae9cbb7359cdf2051, 360d0b423433ec91f787be3bde665606, 603677e75f480cfd74204fce51bd5359, 796f7fc8fad8d003c466a3f53b960815, 8695d5f6c29ed614cc7e97c2ba4c6724, 8af3d7357dfa0f8aa36ba6b01663d6f6, a05cacf8eaf88a4edb7255a00c86d78c, b272925d8b30354bd56e190c0c041139, bbecc4f063e3fd6151247c0507c5bef3, d17b306f903adc2b73f818b128d5cd67, ff2b8d760c6f5acc07476412531886a0 2024-11-27T01:14:44,071 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T01:14:44,071 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T01:14:44,071 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T01:14:44,071 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T01:14:44,071 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T01:14:44,071 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-27T01:14:44,098 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/bbecc4f063e3fd6151247c0507c5bef3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,101 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/info/1fffc740598a40c283668cf8b3ef4cb8 is 138, key is test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3./info:regioninfo/1732670081292/Put/seqid=0 2024-11-27T01:14:44,111 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:44,111 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bbecc4f063e3fd6151247c0507c5bef3: Waiting for close lock at 1732670084065Running coprocessor pre-close hooks at 1732670084065Disabling compacts and flushes for region at 1732670084065Disabling writes for close at 1732670084065Writing region close event to WAL at 1732670084072 (+7 ms)Running coprocessor post-close hooks at 1732670084099 (+27 ms)Closed at 1732670084111 (+12 ms) 2024-11-27T01:14:44,112 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732670076899.bbecc4f063e3fd6151247c0507c5bef3. 2024-11-27T01:14:44,112 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff2b8d760c6f5acc07476412531886a0, disabling compactions & flushes 2024-11-27T01:14:44,112 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:44,112 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/3cfaabbc55e590863a7ecf45872977b3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,112 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:44,112 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. after waiting 0 ms 2024-11-27T01:14:44,113 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:44,113 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:44,113 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3cfaabbc55e590863a7ecf45872977b3: Waiting for close lock at 1732670084059Running coprocessor pre-close hooks at 1732670084060 (+1 ms)Disabling compacts and flushes for region at 1732670084060Disabling writes for close at 1732670084060Writing region close event to WAL at 1732670084081 (+21 ms)Running coprocessor post-close hooks at 1732670084113 (+32 ms)Closed at 1732670084113 2024-11-27T01:14:44,114 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732670076899.3cfaabbc55e590863a7ecf45872977b3. 2024-11-27T01:14:44,114 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing edc306f98fb391a3fdc7623063a1cbf8, disabling compactions & flushes 2024-11-27T01:14:44,114 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:44,114 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:44,114 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. after waiting 0 ms 2024-11-27T01:14:44,114 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:44,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741869_1045 (size=8195) 2024-11-27T01:14:44,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741869_1045 (size=8195) 2024-11-27T01:14:44,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741869_1045 (size=8195) 2024-11-27T01:14:44,141 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/info/1fffc740598a40c283668cf8b3ef4cb8 2024-11-27T01:14:44,145 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/edc306f98fb391a3fdc7623063a1cbf8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,146 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:44,146 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for edc306f98fb391a3fdc7623063a1cbf8: Waiting for close lock at 1732670084114Running coprocessor pre-close hooks at 1732670084114Disabling compacts and flushes for region at 1732670084114Disabling writes for close at 1732670084114Writing region close event to WAL at 1732670084118 (+4 ms)Running coprocessor post-close hooks at 1732670084146 (+28 ms)Closed at 1732670084146 2024-11-27T01:14:44,147 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732670076899.edc306f98fb391a3fdc7623063a1cbf8. 2024-11-27T01:14:44,147 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b85ee4f1fa33a9cbdcb45c32b6176c88, disabling compactions & flushes 2024-11-27T01:14:44,147 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:44,147 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:44,147 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. after waiting 0 ms 2024-11-27T01:14:44,147 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:44,152 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/.tmp/info/1fffc740598a40c283668cf8b3ef4cb8 as hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/info/1fffc740598a40c283668cf8b3ef4cb8 2024-11-27T01:14:44,158 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/ff2b8d760c6f5acc07476412531886a0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,159 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:44,159 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff2b8d760c6f5acc07476412531886a0: Waiting for close lock at 1732670084112Running coprocessor pre-close hooks at 1732670084112Disabling compacts and flushes for region at 1732670084112Disabling writes for close at 1732670084112Writing region close event to WAL at 1732670084116 (+4 ms)Running coprocessor post-close hooks at 1732670084159 (+43 ms)Closed at 1732670084159 2024-11-27T01:14:44,159 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732670076899.ff2b8d760c6f5acc07476412531886a0. 2024-11-27T01:14:44,159 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0caf01005a5fb704a147ee46c7227b3e, disabling compactions & flushes 2024-11-27T01:14:44,159 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:44,159 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:44,159 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. after waiting 0 ms 2024-11-27T01:14:44,159 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:44,166 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/b85ee4f1fa33a9cbdcb45c32b6176c88/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,167 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:44,167 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b85ee4f1fa33a9cbdcb45c32b6176c88: Waiting for close lock at 1732670084147Running coprocessor pre-close hooks at 1732670084147Disabling compacts and flushes for region at 1732670084147Disabling writes for close at 1732670084147Writing region close event to WAL at 1732670084160 (+13 ms)Running coprocessor post-close hooks at 1732670084167 (+7 ms)Closed at 1732670084167 2024-11-27T01:14:44,167 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732670076899.b85ee4f1fa33a9cbdcb45c32b6176c88. 2024-11-27T01:14:44,168 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4a61d50f100ead183465a200984a5e77, disabling compactions & flushes 2024-11-27T01:14:44,168 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:44,168 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:44,168 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. after waiting 0 ms 2024-11-27T01:14:44,168 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:44,169 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/0caf01005a5fb704a147ee46c7227b3e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,169 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/info/1fffc740598a40c283668cf8b3ef4cb8, entries=24, sequenceid=71, filesize=8.0 K 2024-11-27T01:14:44,170 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:44,170 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0caf01005a5fb704a147ee46c7227b3e: Waiting for close lock at 1732670084159Running coprocessor pre-close hooks at 1732670084159Disabling compacts and flushes for region at 1732670084159Disabling writes for close at 1732670084159Writing region close event to WAL at 1732670084161 (+2 ms)Running coprocessor post-close hooks at 1732670084170 (+9 ms)Closed at 1732670084170 2024-11-27T01:14:44,170 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732670076899.0caf01005a5fb704a147ee46c7227b3e. 2024-11-27T01:14:44,170 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2fed3d0dc5a44acae9cbb7359cdf2051, disabling compactions & flushes 2024-11-27T01:14:44,170 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:44,170 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:44,170 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. after waiting 0 ms 2024-11-27T01:14:44,170 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:44,171 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=71, compaction requested=false 2024-11-27T01:14:44,174 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4a61d50f100ead183465a200984a5e77/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,174 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:44,174 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4a61d50f100ead183465a200984a5e77: Waiting for close lock at 1732670084168Running coprocessor pre-close hooks at 1732670084168Disabling compacts and flushes for region at 1732670084168Disabling writes for close at 1732670084168Writing region close event to WAL at 1732670084168Running coprocessor post-close hooks at 1732670084174 (+6 ms)Closed at 1732670084174 2024-11-27T01:14:44,174 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732670076899.4a61d50f100ead183465a200984a5e77. 2024-11-27T01:14:44,175 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fc23d7fa863cb970ac753db9e4e26b26, disabling compactions & flushes 2024-11-27T01:14:44,175 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:44,175 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:44,175 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. after waiting 0 ms 2024-11-27T01:14:44,175 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:44,189 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/fc23d7fa863cb970ac753db9e4e26b26/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,190 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-27T01:14:44,191 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T01:14:44,191 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T01:14:44,191 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732670084071Running coprocessor pre-close hooks at 1732670084071Disabling compacts and flushes for region at 1732670084071Disabling writes for close at 1732670084071Obtaining lock to block concurrent updates at 1732670084071Preparing flush snapshotting stores in 1588230740 at 1732670084071Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732670084078 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732670084085 (+7 ms)Flushing 1588230740/info: creating writer at 1732670084085Flushing 1588230740/info: appending metadata at 1732670084101 (+16 ms)Flushing 1588230740/info: closing flushed file at 1732670084101Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b3340ab: reopening flushed file at 1732670084151 (+50 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=71, compaction requested=false at 1732670084171 (+20 ms)Writing region close event to WAL at 1732670084178 (+7 ms)Running coprocessor post-close hooks at 1732670084191 (+13 ms)Closed at 1732670084191 2024-11-27T01:14:44,191 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T01:14:44,192 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:44,192 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fc23d7fa863cb970ac753db9e4e26b26: Waiting for close lock at 1732670084174Running coprocessor pre-close hooks at 1732670084174Disabling compacts and flushes for region at 1732670084175 (+1 ms)Disabling writes for close at 1732670084175Writing region close event to WAL at 1732670084176 (+1 ms)Running coprocessor post-close hooks at 1732670084192 (+16 ms)Closed at 1732670084192 2024-11-27T01:14:44,192 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732670076899.fc23d7fa863cb970ac753db9e4e26b26. 2024-11-27T01:14:44,192 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/2fed3d0dc5a44acae9cbb7359cdf2051/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,192 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ae4c3f2c0fe2fac2f19395fca402eb5, disabling compactions & flushes 2024-11-27T01:14:44,192 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:44,192 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:44,193 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. after waiting 0 ms 2024-11-27T01:14:44,193 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:44,193 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:44,193 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2fed3d0dc5a44acae9cbb7359cdf2051: Waiting for close lock at 1732670084170Running coprocessor pre-close hooks at 1732670084170Disabling compacts and flushes for region at 1732670084170Disabling writes for close at 1732670084170Writing region close event to WAL at 1732670084171 (+1 ms)Running coprocessor post-close hooks at 1732670084193 (+22 ms)Closed at 1732670084193 2024-11-27T01:14:44,193 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732670076899.2fed3d0dc5a44acae9cbb7359cdf2051. 2024-11-27T01:14:44,193 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a05cacf8eaf88a4edb7255a00c86d78c, disabling compactions & flushes 2024-11-27T01:14:44,193 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:44,193 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:44,193 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. after waiting 0 ms 2024-11-27T01:14:44,194 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:44,202 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/a05cacf8eaf88a4edb7255a00c86d78c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,203 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:44,204 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a05cacf8eaf88a4edb7255a00c86d78c: Waiting for close lock at 1732670084193Running coprocessor pre-close hooks at 1732670084193Disabling compacts and flushes for region at 1732670084193Disabling writes for close at 1732670084193Writing region close event to WAL at 1732670084197 (+4 ms)Running coprocessor post-close hooks at 1732670084203 (+6 ms)Closed at 1732670084203 2024-11-27T01:14:44,204 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732670076899.a05cacf8eaf88a4edb7255a00c86d78c. 2024-11-27T01:14:44,204 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8695d5f6c29ed614cc7e97c2ba4c6724, disabling compactions & flushes 2024-11-27T01:14:44,204 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:44,204 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:44,204 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. after waiting 0 ms 2024-11-27T01:14:44,204 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:44,211 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8ae4c3f2c0fe2fac2f19395fca402eb5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,212 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:44,212 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ae4c3f2c0fe2fac2f19395fca402eb5: Waiting for close lock at 1732670084192Running coprocessor pre-close hooks at 1732670084192Disabling compacts and flushes for region at 1732670084192Disabling writes for close at 1732670084193 (+1 ms)Writing region close event to WAL at 1732670084198 (+5 ms)Running coprocessor post-close hooks at 1732670084212 (+14 ms)Closed at 1732670084212 2024-11-27T01:14:44,212 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732670076899.8ae4c3f2c0fe2fac2f19395fca402eb5. 2024-11-27T01:14:44,213 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing adbccc292c5b7e61d47646b64c419a44, disabling compactions & flushes 2024-11-27T01:14:44,213 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:44,213 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:44,213 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. after waiting 0 ms 2024-11-27T01:14:44,213 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:44,226 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/adbccc292c5b7e61d47646b64c419a44/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,226 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8695d5f6c29ed614cc7e97c2ba4c6724/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,227 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:44,227 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for adbccc292c5b7e61d47646b64c419a44: Waiting for close lock at 1732670084212Running coprocessor pre-close hooks at 1732670084212Disabling compacts and flushes for region at 1732670084212Disabling writes for close at 1732670084213 (+1 ms)Writing region close event to WAL at 1732670084217 (+4 ms)Running coprocessor post-close hooks at 1732670084226 (+9 ms)Closed at 1732670084227 (+1 ms) 2024-11-27T01:14:44,227 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732670076899.adbccc292c5b7e61d47646b64c419a44. 2024-11-27T01:14:44,227 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:44,227 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 79f1ff709eae178b11e9bcfa1e4131b2, disabling compactions & flushes 2024-11-27T01:14:44,227 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:44,227 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8695d5f6c29ed614cc7e97c2ba4c6724: Waiting for close lock at 1732670084204Running coprocessor pre-close hooks at 1732670084204Disabling compacts and flushes for region at 1732670084204Disabling writes for close at 1732670084204Writing region close event to WAL at 1732670084208 (+4 ms)Running coprocessor post-close hooks at 1732670084227 (+19 ms)Closed at 1732670084227 2024-11-27T01:14:44,227 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:44,227 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. after waiting 0 ms 2024-11-27T01:14:44,227 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732670076899.8695d5f6c29ed614cc7e97c2ba4c6724. 2024-11-27T01:14:44,227 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:44,227 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8af3d7357dfa0f8aa36ba6b01663d6f6, disabling compactions & flushes 2024-11-27T01:14:44,227 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:44,228 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:44,228 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. after waiting 0 ms 2024-11-27T01:14:44,228 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:44,233 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/79f1ff709eae178b11e9bcfa1e4131b2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,234 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:44,234 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 79f1ff709eae178b11e9bcfa1e4131b2: Waiting for close lock at 1732670084227Running coprocessor pre-close hooks at 1732670084227Disabling compacts and flushes for region at 1732670084227Disabling writes for close at 1732670084227Writing region close event to WAL at 1732670084228 (+1 ms)Running coprocessor post-close hooks at 1732670084233 (+5 ms)Closed at 1732670084234 (+1 ms) 2024-11-27T01:14:44,234 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732670076899.79f1ff709eae178b11e9bcfa1e4131b2. 2024-11-27T01:14:44,234 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 968f86830e42ee358d5e11672847740f, disabling compactions & flushes 2024-11-27T01:14:44,234 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:44,234 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:44,234 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. after waiting 0 ms 2024-11-27T01:14:44,234 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:44,237 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8af3d7357dfa0f8aa36ba6b01663d6f6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,238 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:44,238 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8af3d7357dfa0f8aa36ba6b01663d6f6: Waiting for close lock at 1732670084227Running coprocessor pre-close hooks at 1732670084227Disabling compacts and flushes for region at 1732670084227Disabling writes for close at 1732670084228 (+1 ms)Writing region close event to WAL at 1732670084228Running coprocessor post-close hooks at 1732670084238 (+10 ms)Closed at 1732670084238 2024-11-27T01:14:44,238 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732670076899.8af3d7357dfa0f8aa36ba6b01663d6f6. 2024-11-27T01:14:44,238 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b272925d8b30354bd56e190c0c041139, disabling compactions & flushes 2024-11-27T01:14:44,238 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:44,238 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:44,239 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. after waiting 0 ms 2024-11-27T01:14:44,239 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:44,246 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/968f86830e42ee358d5e11672847740f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,246 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/b272925d8b30354bd56e190c0c041139/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,247 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 968f86830e42ee358d5e11672847740f: Waiting for close lock at 1732670084234Running coprocessor pre-close hooks at 1732670084234Disabling compacts and flushes for region at 1732670084234Disabling writes for close at 1732670084234Writing region close event to WAL at 1732670084240 (+6 ms)Running coprocessor post-close hooks at 1732670084246 (+6 ms)Closed at 1732670084247 (+1 ms) 2024-11-27T01:14:44,247 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732670076899.968f86830e42ee358d5e11672847740f. 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b272925d8b30354bd56e190c0c041139: Waiting for close lock at 1732670084238Running coprocessor pre-close hooks at 1732670084238Disabling compacts and flushes for region at 1732670084238Disabling writes for close at 1732670084239 (+1 ms)Writing region close event to WAL at 1732670084241 (+2 ms)Running coprocessor post-close hooks at 1732670084247 (+6 ms)Closed at 1732670084247 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732670076899.b272925d8b30354bd56e190c0c041139. 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4b3cb4626ebbddc356f324f4faf22f80, disabling compactions & flushes 2024-11-27T01:14:44,247 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 360d0b423433ec91f787be3bde665606, disabling compactions & flushes 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:44,247 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. after waiting 0 ms 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. after waiting 0 ms 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:44,247 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:44,252 DEBUG [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(1351): Waiting on 335860f4847882dab27e1db49323c60b, 4b3cb4626ebbddc356f324f4faf22f80, 8e92cc2f48378cc4a64c15f36bcf6052, eedb1cdc1fad3f780344bbfed739194f 2024-11-27T01:14:44,255 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/4b3cb4626ebbddc356f324f4faf22f80/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,255 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/360d0b423433ec91f787be3bde665606/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,256 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4b3cb4626ebbddc356f324f4faf22f80: Waiting for close lock at 1732670084247Running coprocessor pre-close hooks at 1732670084247Disabling compacts and flushes for region at 1732670084247Disabling writes for close at 1732670084247Writing region close event to WAL at 1732670084248 (+1 ms)Running coprocessor post-close hooks at 1732670084256 (+8 ms)Closed at 1732670084256 2024-11-27T01:14:44,256 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732670076899.4b3cb4626ebbddc356f324f4faf22f80. 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 360d0b423433ec91f787be3bde665606: Waiting for close lock at 1732670084247Running coprocessor pre-close hooks at 1732670084247Disabling compacts and flushes for region at 1732670084247Disabling writes for close at 1732670084247Writing region close event to WAL at 1732670084248 (+1 ms)Running coprocessor post-close hooks at 1732670084256 (+8 ms)Closed at 1732670084256 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8e92cc2f48378cc4a64c15f36bcf6052, disabling compactions & flushes 2024-11-27T01:14:44,256 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732670076899.360d0b423433ec91f787be3bde665606. 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. after waiting 0 ms 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 796f7fc8fad8d003c466a3f53b960815, disabling compactions & flushes 2024-11-27T01:14:44,256 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. after waiting 0 ms 2024-11-27T01:14:44,256 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:44,262 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/8e92cc2f48378cc4a64c15f36bcf6052/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,262 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/796f7fc8fad8d003c466a3f53b960815/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,262 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:44,262 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8e92cc2f48378cc4a64c15f36bcf6052: Waiting for close lock at 1732670084256Running coprocessor pre-close hooks at 1732670084256Disabling compacts and flushes for region at 1732670084256Disabling writes for close at 1732670084256Writing region close event to WAL at 1732670084257 (+1 ms)Running coprocessor post-close hooks at 1732670084262 (+5 ms)Closed at 1732670084262 2024-11-27T01:14:44,263 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732670076899.8e92cc2f48378cc4a64c15f36bcf6052. 2024-11-27T01:14:44,263 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eedb1cdc1fad3f780344bbfed739194f, disabling compactions & flushes 2024-11-27T01:14:44,263 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:44,263 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:44,263 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. after waiting 0 ms 2024-11-27T01:14:44,263 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:44,264 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:44,264 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 796f7fc8fad8d003c466a3f53b960815: Waiting for close lock at 1732670084256Running coprocessor pre-close hooks at 1732670084256Disabling compacts and flushes for region at 1732670084256Disabling writes for close at 1732670084256Writing region close event to WAL at 1732670084257 (+1 ms)Running coprocessor post-close hooks at 1732670084263 (+6 ms)Closed at 1732670084264 (+1 ms) 2024-11-27T01:14:44,264 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732670076899.796f7fc8fad8d003c466a3f53b960815. 2024-11-27T01:14:44,264 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d17b306f903adc2b73f818b128d5cd67, disabling compactions & flushes 2024-11-27T01:14:44,264 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:44,264 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:44,264 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. after waiting 0 ms 2024-11-27T01:14:44,264 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:44,270 DEBUG [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(1351): Waiting on 1229242f433b46943919d6643a95d4a4, 603677e75f480cfd74204fce51bd5359, d17b306f903adc2b73f818b128d5cd67 2024-11-27T01:14:44,276 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/eedb1cdc1fad3f780344bbfed739194f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,278 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:44,278 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eedb1cdc1fad3f780344bbfed739194f: Waiting for close lock at 1732670084263Running coprocessor pre-close hooks at 1732670084263Disabling compacts and flushes for region at 1732670084263Disabling writes for close at 1732670084263Writing region close event to WAL at 1732670084264 (+1 ms)Running coprocessor post-close hooks at 1732670084278 (+14 ms)Closed at 1732670084278 2024-11-27T01:14:44,278 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732670076899.eedb1cdc1fad3f780344bbfed739194f. 2024-11-27T01:14:44,278 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 335860f4847882dab27e1db49323c60b, disabling compactions & flushes 2024-11-27T01:14:44,278 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:44,279 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:44,279 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. after waiting 0 ms 2024-11-27T01:14:44,279 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:44,281 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/d17b306f903adc2b73f818b128d5cd67/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,281 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:44,282 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d17b306f903adc2b73f818b128d5cd67: Waiting for close lock at 1732670084264Running coprocessor pre-close hooks at 1732670084264Disabling compacts and flushes for region at 1732670084264Disabling writes for close at 1732670084264Writing region close event to WAL at 1732670084265 (+1 ms)Running coprocessor post-close hooks at 1732670084281 (+16 ms)Closed at 1732670084281 2024-11-27T01:14:44,282 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732670076899.d17b306f903adc2b73f818b128d5cd67. 2024-11-27T01:14:44,282 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1229242f433b46943919d6643a95d4a4, disabling compactions & flushes 2024-11-27T01:14:44,282 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:44,282 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:44,282 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. after waiting 0 ms 2024-11-27T01:14:44,282 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:44,288 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/335860f4847882dab27e1db49323c60b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,289 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:44,289 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/1229242f433b46943919d6643a95d4a4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,289 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 335860f4847882dab27e1db49323c60b: Waiting for close lock at 1732670084278Running coprocessor pre-close hooks at 1732670084278Disabling compacts and flushes for region at 1732670084278Disabling writes for close at 1732670084279 (+1 ms)Writing region close event to WAL at 1732670084279Running coprocessor post-close hooks at 1732670084289 (+10 ms)Closed at 1732670084289 2024-11-27T01:14:44,289 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732670076899.335860f4847882dab27e1db49323c60b. 2024-11-27T01:14:44,289 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:44,289 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1229242f433b46943919d6643a95d4a4: Waiting for close lock at 1732670084282Running coprocessor pre-close hooks at 1732670084282Disabling compacts and flushes for region at 1732670084282Disabling writes for close at 1732670084282Writing region close event to WAL at 1732670084285 (+3 ms)Running coprocessor post-close hooks at 1732670084289 (+4 ms)Closed at 1732670084289 2024-11-27T01:14:44,289 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732670076899.1229242f433b46943919d6643a95d4a4. 2024-11-27T01:14:44,290 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 603677e75f480cfd74204fce51bd5359, disabling compactions & flushes 2024-11-27T01:14:44,290 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:44,290 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:44,290 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. after waiting 0 ms 2024-11-27T01:14:44,290 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:44,293 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/data/default/test/603677e75f480cfd74204fce51bd5359/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:14:44,294 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:44,294 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 603677e75f480cfd74204fce51bd5359: Waiting for close lock at 1732670084290Running coprocessor pre-close hooks at 1732670084290Disabling compacts and flushes for region at 1732670084290Disabling writes for close at 1732670084290Writing region close event to WAL at 1732670084290Running coprocessor post-close hooks at 1732670084294 (+4 ms)Closed at 1732670084294 2024-11-27T01:14:44,294 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732670076899.603677e75f480cfd74204fce51bd5359. 2024-11-27T01:14:44,453 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(976): stopping server 7d6f26a4c136,39915,1732670073690; all regions closed. 2024-11-27T01:14:44,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741833_1009 (size=7064) 2024-11-27T01:14:44,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741833_1009 (size=7064) 2024-11-27T01:14:44,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741833_1009 (size=7064) 2024-11-27T01:14:44,461 DEBUG [RS:0;7d6f26a4c136:39915 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs 2024-11-27T01:14:44,461 INFO [RS:0;7d6f26a4c136:39915 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C39915%2C1732670073690:(num 1732670075807) 2024-11-27T01:14:44,462 DEBUG [RS:0;7d6f26a4c136:39915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:44,462 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:14:44,462 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:14:44,462 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.ChoreService(370): Chore service for: regionserver/7d6f26a4c136:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T01:14:44,462 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T01:14:44,462 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T01:14:44,462 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T01:14:44,462 INFO [regionserver/7d6f26a4c136:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:14:44,462 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:14:44,463 INFO [RS:0;7d6f26a4c136:39915 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39915 2024-11-27T01:14:44,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:14:44,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,39915,1732670073690 2024-11-27T01:14:44,465 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:14:44,466 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,39915,1732670073690] 2024-11-27T01:14:44,469 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,39915,1732670073690 already deleted, retry=false 2024-11-27T01:14:44,470 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d6f26a4c136,39915,1732670073690 expired; onlineServers=1 2024-11-27T01:14:44,470 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(976): stopping server 7d6f26a4c136,40647,1732670073826; all regions closed. 2024-11-27T01:14:44,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741865_1041 (size=4204) 2024-11-27T01:14:44,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741865_1041 (size=4204) 2024-11-27T01:14:44,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741865_1041 (size=4204) 2024-11-27T01:14:44,478 DEBUG [RS:1;7d6f26a4c136:40647 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs 2024-11-27T01:14:44,478 INFO [RS:1;7d6f26a4c136:40647 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C40647%2C1732670073826.meta:.meta(num 1732670080179) 2024-11-27T01:14:44,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741835_1011 (size=7074) 2024-11-27T01:14:44,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741835_1011 (size=7074) 2024-11-27T01:14:44,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741835_1011 (size=7074) 2024-11-27T01:14:44,487 DEBUG [RS:1;7d6f26a4c136:40647 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/oldWALs 2024-11-27T01:14:44,487 INFO [RS:1;7d6f26a4c136:40647 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C40647%2C1732670073826:(num 1732670075803) 2024-11-27T01:14:44,487 DEBUG [RS:1;7d6f26a4c136:40647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:14:44,487 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:14:44,487 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:14:44,488 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.ChoreService(370): Chore service for: regionserver/7d6f26a4c136:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T01:14:44,488 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:14:44,488 INFO [regionserver/7d6f26a4c136:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:14:44,488 INFO [RS:1;7d6f26a4c136:40647 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40647 2024-11-27T01:14:44,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:14:44,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,40647,1732670073826 2024-11-27T01:14:44,491 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:14:44,492 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,40647,1732670073826] 2024-11-27T01:14:44,495 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,40647,1732670073826 already deleted, retry=false 2024-11-27T01:14:44,495 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d6f26a4c136,40647,1732670073826 expired; onlineServers=0 2024-11-27T01:14:44,495 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7d6f26a4c136,33239,1732670072670' ***** 2024-11-27T01:14:44,496 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T01:14:44,496 INFO [M:0;7d6f26a4c136:33239 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:14:44,496 INFO [M:0;7d6f26a4c136:33239 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:14:44,496 DEBUG [M:0;7d6f26a4c136:33239 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T01:14:44,496 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T01:14:44,496 DEBUG [M:0;7d6f26a4c136:33239 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T01:14:44,496 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732670075321 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732670075321,5,FailOnTimeoutGroup] 2024-11-27T01:14:44,496 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732670075322 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732670075322,5,FailOnTimeoutGroup] 2024-11-27T01:14:44,496 INFO [M:0;7d6f26a4c136:33239 {}] hbase.ChoreService(370): Chore service for: master/7d6f26a4c136:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T01:14:44,497 INFO [M:0;7d6f26a4c136:33239 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:14:44,497 DEBUG [M:0;7d6f26a4c136:33239 {}] master.HMaster(1795): Stopping service threads 2024-11-27T01:14:44,497 INFO [M:0;7d6f26a4c136:33239 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T01:14:44,497 INFO [M:0;7d6f26a4c136:33239 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:14:44,497 INFO [M:0;7d6f26a4c136:33239 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:14:44,498 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T01:14:44,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T01:14:44,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:14:44,499 DEBUG [M:0;7d6f26a4c136:33239 {}] zookeeper.ZKUtil(347): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T01:14:44,499 WARN [M:0;7d6f26a4c136:33239 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T01:14:44,500 INFO [M:0;7d6f26a4c136:33239 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/.lastflushedseqids 2024-11-27T01:14:44,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741870_1046 (size=1832) 2024-11-27T01:14:44,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741870_1046 (size=1832) 2024-11-27T01:14:44,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741870_1046 (size=1832) 2024-11-27T01:14:44,524 INFO [M:0;7d6f26a4c136:33239 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:14:44,525 INFO [M:0;7d6f26a4c136:33239 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T01:14:44,525 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T01:14:44,525 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:14:44,525 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:14:44,525 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T01:14:44,525 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:14:44,525 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=250.12 KB heapSize=303.95 KB 2024-11-27T01:14:44,543 DEBUG [M:0;7d6f26a4c136:33239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/38fc292de59749be842a8ef51320cf99 is 82, key is hbase:meta,,1/info:regioninfo/1732670080879/Put/seqid=0 2024-11-27T01:14:44,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741871_1047 (size=6071) 2024-11-27T01:14:44,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741871_1047 (size=6071) 2024-11-27T01:14:44,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741871_1047 (size=6071) 2024-11-27T01:14:44,551 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/38fc292de59749be842a8ef51320cf99 2024-11-27T01:14:44,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:14:44,569 INFO [RS:0;7d6f26a4c136:39915 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:14:44,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39915-0x10156fd0ddf0001, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:14:44,569 INFO [RS:0;7d6f26a4c136:39915 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,39915,1732670073690; zookeeper connection closed. 2024-11-27T01:14:44,569 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75ff08eb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75ff08eb 2024-11-27T01:14:44,576 DEBUG [M:0;7d6f26a4c136:33239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ac0b1aee32ea4a7b89317b0bdddeb370 is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732670083974/Put/seqid=0 2024-11-27T01:14:44,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741872_1048 (size=12802) 2024-11-27T01:14:44,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741872_1048 (size=12802) 2024-11-27T01:14:44,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741872_1048 (size=12802) 2024-11-27T01:14:44,585 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.84 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ac0b1aee32ea4a7b89317b0bdddeb370 2024-11-27T01:14:44,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:14:44,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40647-0x10156fd0ddf0002, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:14:44,593 INFO [RS:1;7d6f26a4c136:40647 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:14:44,593 INFO [RS:1;7d6f26a4c136:40647 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,40647,1732670073826; zookeeper connection closed. 2024-11-27T01:14:44,593 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@12468e0c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@12468e0c 2024-11-27T01:14:44,594 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-27T01:14:44,607 DEBUG [M:0;7d6f26a4c136:33239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b2aa664548554e108bd595d982ff0a69 is 69, key is 7d6f26a4c136,39915,1732670073690/rs:state/1732670075376/Put/seqid=0 2024-11-27T01:14:44,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741873_1049 (size=5442) 2024-11-27T01:14:44,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741873_1049 (size=5442) 2024-11-27T01:14:44,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741873_1049 (size=5442) 2024-11-27T01:14:44,616 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b2aa664548554e108bd595d982ff0a69 2024-11-27T01:14:44,622 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b2aa664548554e108bd595d982ff0a69 2024-11-27T01:14:44,639 DEBUG [M:0;7d6f26a4c136:33239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7dbe2ad2cbf34b2aa460bf3df4506d13 is 52, key is load_balancer_on/state:d/1732670079328/Put/seqid=0 2024-11-27T01:14:44,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741874_1050 (size=5058) 2024-11-27T01:14:44,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741874_1050 (size=5058) 2024-11-27T01:14:44,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741874_1050 (size=5058) 2024-11-27T01:14:44,647 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7dbe2ad2cbf34b2aa460bf3df4506d13 2024-11-27T01:14:44,656 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/38fc292de59749be842a8ef51320cf99 as hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/38fc292de59749be842a8ef51320cf99 2024-11-27T01:14:44,664 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/38fc292de59749be842a8ef51320cf99, entries=14, sequenceid=575, filesize=5.9 K 2024-11-27T01:14:44,666 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ac0b1aee32ea4a7b89317b0bdddeb370 as hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ac0b1aee32ea4a7b89317b0bdddeb370 2024-11-27T01:14:44,672 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ac0b1aee32ea4a7b89317b0bdddeb370, entries=79, sequenceid=575, filesize=12.5 K 2024-11-27T01:14:44,673 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b2aa664548554e108bd595d982ff0a69 as hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b2aa664548554e108bd595d982ff0a69 2024-11-27T01:14:44,679 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b2aa664548554e108bd595d982ff0a69 2024-11-27T01:14:44,679 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b2aa664548554e108bd595d982ff0a69, entries=3, sequenceid=575, filesize=5.3 K 2024-11-27T01:14:44,681 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7dbe2ad2cbf34b2aa460bf3df4506d13 as hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7dbe2ad2cbf34b2aa460bf3df4506d13 2024-11-27T01:14:44,686 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33923/user/jenkins/test-data/260ac3d6-52f3-be51-6305-b7d9c2e4c06d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7dbe2ad2cbf34b2aa460bf3df4506d13, entries=1, sequenceid=575, filesize=4.9 K 2024-11-27T01:14:44,687 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.HRegion(3140): Finished flush of dataSize ~250.12 KB/256120, heapSize ~303.89 KB/311184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=575, compaction requested=false 2024-11-27T01:14:44,688 INFO [M:0;7d6f26a4c136:33239 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:14:44,688 DEBUG [M:0;7d6f26a4c136:33239 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732670084525Disabling compacts and flushes for region at 1732670084525Disabling writes for close at 1732670084525Obtaining lock to block concurrent updates at 1732670084525Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732670084525Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=256120, getHeapSize=311184, getOffHeapSize=0, getCellsCount=714 at 1732670084525Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732670084526 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732670084526Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732670084543 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732670084543Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732670084558 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732670084576 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732670084576Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732670084590 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732670084606 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732670084606Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732670084622 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732670084638 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732670084638Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dc7cb0a: reopening flushed file at 1732670084655 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75f6e172: reopening flushed file at 1732670084664 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d7c8aa4: reopening flushed file at 1732670084672 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@560e260f: reopening flushed file at 1732670084680 (+8 ms)Finished flush of dataSize ~250.12 KB/256120, heapSize ~303.89 KB/311184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=575, compaction requested=false at 1732670084687 (+7 ms)Writing region close event to WAL at 1732670084688 (+1 ms)Closed at 1732670084688 2024-11-27T01:14:44,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40703 is added to blk_1073741830_1006 (size=294504) 2024-11-27T01:14:44,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46391 is added to blk_1073741830_1006 (size=294504) 2024-11-27T01:14:44,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741830_1006 (size=294504) 2024-11-27T01:14:44,692 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:14:44,692 INFO [M:0;7d6f26a4c136:33239 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T01:14:44,692 INFO [M:0;7d6f26a4c136:33239 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33239 2024-11-27T01:14:44,692 INFO [M:0;7d6f26a4c136:33239 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:14:44,794 INFO [M:0;7d6f26a4c136:33239 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:14:44,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:14:44,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33239-0x10156fd0ddf0000, quorum=127.0.0.1:54756, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:14:44,798 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3351f4c8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:14:44,800 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51924127{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:14:44,801 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:14:44,801 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a0c767{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:14:44,801 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@609b663f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.log.dir/,STOPPED} 2024-11-27T01:14:44,803 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:14:44,803 WARN [BP-1607804123-172.17.0.2-1732670067970 heartbeating to localhost/127.0.0.1:33923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:14:44,803 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:14:44,803 WARN [BP-1607804123-172.17.0.2-1732670067970 heartbeating to localhost/127.0.0.1:33923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1607804123-172.17.0.2-1732670067970 (Datanode Uuid 2b23d955-eed6-4dac-bb99-b526474368cc) service to localhost/127.0.0.1:33923 2024-11-27T01:14:44,805 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data5/current/BP-1607804123-172.17.0.2-1732670067970 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:14:44,805 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data6/current/BP-1607804123-172.17.0.2-1732670067970 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:14:44,805 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:14:44,807 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2dca9824{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:14:44,807 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@46b93c21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:14:44,807 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:14:44,808 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b427afd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:14:44,808 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a294b42{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.log.dir/,STOPPED} 2024-11-27T01:14:44,809 WARN [BP-1607804123-172.17.0.2-1732670067970 heartbeating to localhost/127.0.0.1:33923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:14:44,809 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:14:44,809 WARN [BP-1607804123-172.17.0.2-1732670067970 heartbeating to localhost/127.0.0.1:33923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1607804123-172.17.0.2-1732670067970 (Datanode Uuid 02f1b4ec-72df-4ada-929d-8dd81b7b18e0) service to localhost/127.0.0.1:33923 2024-11-27T01:14:44,809 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:14:44,810 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data3/current/BP-1607804123-172.17.0.2-1732670067970 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:14:44,810 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data4/current/BP-1607804123-172.17.0.2-1732670067970 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:14:44,810 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:14:44,812 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a0ce16d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:14:44,812 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1cafe9ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:14:44,812 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:14:44,812 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62c0eab0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:14:44,812 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18a3047c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.log.dir/,STOPPED} 2024-11-27T01:14:44,814 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:14:44,814 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:14:44,814 WARN [BP-1607804123-172.17.0.2-1732670067970 heartbeating to localhost/127.0.0.1:33923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:14:44,814 WARN [BP-1607804123-172.17.0.2-1732670067970 heartbeating to localhost/127.0.0.1:33923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1607804123-172.17.0.2-1732670067970 (Datanode Uuid 7ef4efb0-4b93-4b26-aae3-73e7ee13a4f2) service to localhost/127.0.0.1:33923 2024-11-27T01:14:44,815 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data1/current/BP-1607804123-172.17.0.2-1732670067970 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:14:44,815 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/cluster_b2ea9852-3b5c-d8a9-ec02-99f06ab68ab2/data/data2/current/BP-1607804123-172.17.0.2-1732670067970 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:14:44,815 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:14:44,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@302cfe51{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T01:14:44,825 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c1eb5a9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:14:44,825 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:14:44,825 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@378c34a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:14:44,825 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@952689a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/770e3b78-dd8c-d372-2434-6fb683dfada1/hadoop.log.dir/,STOPPED} 2024-11-27T01:14:44,837 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T01:14:44,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down