2024-11-21 22:00:14,994 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-21 22:00:15,005 main DEBUG Took 0.009486 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 22:00:15,005 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 22:00:15,005 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 22:00:15,006 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 22:00:15,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,014 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 22:00:15,026 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,028 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,028 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,029 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,029 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,030 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,031 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,032 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,033 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,033 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,034 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,035 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,035 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,035 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,036 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 22:00:15,037 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,037 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 22:00:15,039 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 22:00:15,040 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 22:00:15,041 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 22:00:15,042 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 22:00:15,043 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 22:00:15,044 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 22:00:15,051 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 22:00:15,053 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 22:00:15,055 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 22:00:15,055 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 22:00:15,056 main DEBUG createAppenders(={Console}) 2024-11-21 22:00:15,057 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-21 22:00:15,057 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-21 22:00:15,058 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-21 22:00:15,059 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 22:00:15,059 main DEBUG OutputStream closed 2024-11-21 22:00:15,059 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 22:00:15,060 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 22:00:15,060 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-21 22:00:15,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 22:00:15,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 22:00:15,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 22:00:15,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 22:00:15,155 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 22:00:15,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 22:00:15,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 22:00:15,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 22:00:15,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 22:00:15,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 22:00:15,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 22:00:15,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 22:00:15,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 22:00:15,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 22:00:15,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 22:00:15,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 22:00:15,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 22:00:15,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 22:00:15,165 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 22:00:15,165 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-21 22:00:15,166 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 22:00:15,167 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-21T22:00:15,179 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-21 22:00:15,181 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 22:00:15,181 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T22:00:15,438 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8 2024-11-21T22:00:15,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T22:00:15,498 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f, deleteOnExit=true 2024-11-21T22:00:15,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T22:00:15,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/test.cache.data in system properties and HBase conf 2024-11-21T22:00:15,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T22:00:15,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/hadoop.log.dir in system properties and HBase conf 2024-11-21T22:00:15,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T22:00:15,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T22:00:15,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T22:00:15,595 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T22:00:15,681 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T22:00:15,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T22:00:15,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T22:00:15,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T22:00:15,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T22:00:15,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T22:00:15,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T22:00:15,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T22:00:15,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T22:00:15,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T22:00:15,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/nfs.dump.dir in system properties and HBase conf 2024-11-21T22:00:15,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/java.io.tmpdir in system properties and HBase conf 2024-11-21T22:00:15,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T22:00:15,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T22:00:15,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T22:00:16,643 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T22:00:16,715 INFO [Time-limited test {}] log.Log(170): Logging initialized @2404ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T22:00:16,787 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T22:00:16,854 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T22:00:16,875 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T22:00:16,875 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T22:00:16,876 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T22:00:16,887 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T22:00:16,890 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/hadoop.log.dir/,AVAILABLE} 2024-11-21T22:00:16,891 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T22:00:17,070 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/java.io.tmpdir/jetty-localhost-37551-hadoop-hdfs-3_4_1-tests_jar-_-any-14305988073676688864/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T22:00:17,077 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:37551} 2024-11-21T22:00:17,077 INFO [Time-limited test {}] server.Server(415): Started @2767ms 2024-11-21T22:00:17,594 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T22:00:17,602 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T22:00:17,604 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T22:00:17,604 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T22:00:17,604 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T22:00:17,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/hadoop.log.dir/,AVAILABLE} 2024-11-21T22:00:17,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T22:00:17,703 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/java.io.tmpdir/jetty-localhost-39647-hadoop-hdfs-3_4_1-tests_jar-_-any-15517923339139696703/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T22:00:17,704 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:39647} 2024-11-21T22:00:17,704 INFO [Time-limited test {}] server.Server(415): Started @3394ms 2024-11-21T22:00:17,755 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T22:00:17,857 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T22:00:17,864 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T22:00:17,865 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T22:00:17,866 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T22:00:17,866 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T22:00:17,866 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/hadoop.log.dir/,AVAILABLE} 2024-11-21T22:00:17,867 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T22:00:17,984 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/java.io.tmpdir/jetty-localhost-43781-hadoop-hdfs-3_4_1-tests_jar-_-any-13081360617602378903/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T22:00:17,984 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:43781} 2024-11-21T22:00:17,985 INFO [Time-limited test {}] server.Server(415): Started @3675ms 2024-11-21T22:00:17,987 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T22:00:18,021 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T22:00:18,028 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T22:00:18,031 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T22:00:18,031 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T22:00:18,031 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T22:00:18,034 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/hadoop.log.dir/,AVAILABLE} 2024-11-21T22:00:18,035 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T22:00:18,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/java.io.tmpdir/jetty-localhost-35845-hadoop-hdfs-3_4_1-tests_jar-_-any-7653159204629685484/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T22:00:18,139 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:35845} 2024-11-21T22:00:18,140 INFO [Time-limited test {}] server.Server(415): Started @3830ms 2024-11-21T22:00:18,142 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T22:00:19,042 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f/data/data1/current/BP-1060837308-172.17.0.2-1732226416255/current, will proceed with Du for space computation calculation, 2024-11-21T22:00:19,042 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f/data/data4/current/BP-1060837308-172.17.0.2-1732226416255/current, will proceed with Du for space computation calculation, 2024-11-21T22:00:19,042 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f/data/data3/current/BP-1060837308-172.17.0.2-1732226416255/current, will proceed with Du for space computation calculation, 2024-11-21T22:00:19,042 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f/data/data2/current/BP-1060837308-172.17.0.2-1732226416255/current, will proceed with Du for space computation calculation, 2024-11-21T22:00:19,077 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T22:00:19,078 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T22:00:19,087 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f/data/data5/current/BP-1060837308-172.17.0.2-1732226416255/current, will proceed with Du for space computation calculation, 2024-11-21T22:00:19,091 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f/data/data6/current/BP-1060837308-172.17.0.2-1732226416255/current, will proceed with Du for space computation calculation, 2024-11-21T22:00:19,117 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T22:00:19,124 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5bdd2ef420eeed3 with lease ID 0xda84ec3abc2caf68: Processing first storage report for DS-e91666bc-b006-4fed-88f2-dd54fc70cc22 from datanode DatanodeRegistration(127.0.0.1:34171, datanodeUuid=dd402c1e-6352-435c-aa78-6375add8bcaf, infoPort=39043, infoSecurePort=0, ipcPort=45283, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255) 2024-11-21T22:00:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5bdd2ef420eeed3 with lease ID 0xda84ec3abc2caf68: from storage DS-e91666bc-b006-4fed-88f2-dd54fc70cc22 node DatanodeRegistration(127.0.0.1:34171, datanodeUuid=dd402c1e-6352-435c-aa78-6375add8bcaf, infoPort=39043, infoSecurePort=0, ipcPort=45283, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T22:00:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x39753e857949cbd4 with lease ID 0xda84ec3abc2caf69: Processing first storage report for DS-c564ed92-b78e-46e6-b4be-1100870e1511 from datanode DatanodeRegistration(127.0.0.1:37045, datanodeUuid=6e0cc054-5acb-4755-8bfc-34a82c03496a, infoPort=43385, infoSecurePort=0, ipcPort=33723, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255) 2024-11-21T22:00:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x39753e857949cbd4 with lease ID 0xda84ec3abc2caf69: from storage DS-c564ed92-b78e-46e6-b4be-1100870e1511 node DatanodeRegistration(127.0.0.1:37045, datanodeUuid=6e0cc054-5acb-4755-8bfc-34a82c03496a, infoPort=43385, infoSecurePort=0, ipcPort=33723, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T22:00:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc7c5dae8ed631b6f with lease ID 0xda84ec3abc2caf67: Processing first storage report for DS-d8a9d7a5-362c-4f09-b1ac-e85456762592 from datanode DatanodeRegistration(127.0.0.1:43557, datanodeUuid=95d3f247-9f1f-4136-961a-8b174c96f4d7, infoPort=46549, infoSecurePort=0, ipcPort=44203, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255) 2024-11-21T22:00:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc7c5dae8ed631b6f with lease ID 0xda84ec3abc2caf67: from storage DS-d8a9d7a5-362c-4f09-b1ac-e85456762592 node DatanodeRegistration(127.0.0.1:43557, datanodeUuid=95d3f247-9f1f-4136-961a-8b174c96f4d7, infoPort=46549, infoSecurePort=0, ipcPort=44203, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T22:00:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5bdd2ef420eeed3 with lease ID 0xda84ec3abc2caf68: Processing first storage report for DS-c2fccaa0-146c-4aab-a9e9-c1cd5ad5a51e from datanode DatanodeRegistration(127.0.0.1:34171, datanodeUuid=dd402c1e-6352-435c-aa78-6375add8bcaf, infoPort=39043, infoSecurePort=0, ipcPort=45283, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255) 2024-11-21T22:00:19,127 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5bdd2ef420eeed3 with lease ID 0xda84ec3abc2caf68: from storage DS-c2fccaa0-146c-4aab-a9e9-c1cd5ad5a51e node DatanodeRegistration(127.0.0.1:34171, datanodeUuid=dd402c1e-6352-435c-aa78-6375add8bcaf, infoPort=39043, infoSecurePort=0, ipcPort=45283, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T22:00:19,127 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x39753e857949cbd4 with lease ID 0xda84ec3abc2caf69: Processing first storage report for DS-ee9bf480-999c-4dee-b36c-a36d321bdd1e from datanode DatanodeRegistration(127.0.0.1:37045, datanodeUuid=6e0cc054-5acb-4755-8bfc-34a82c03496a, infoPort=43385, infoSecurePort=0, ipcPort=33723, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255) 2024-11-21T22:00:19,127 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x39753e857949cbd4 with lease ID 0xda84ec3abc2caf69: from storage DS-ee9bf480-999c-4dee-b36c-a36d321bdd1e node DatanodeRegistration(127.0.0.1:37045, datanodeUuid=6e0cc054-5acb-4755-8bfc-34a82c03496a, infoPort=43385, infoSecurePort=0, ipcPort=33723, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T22:00:19,127 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc7c5dae8ed631b6f with lease ID 0xda84ec3abc2caf67: Processing first storage report for DS-f3a73415-0859-4728-87c4-f86f9b8f7ae2 from datanode DatanodeRegistration(127.0.0.1:43557, datanodeUuid=95d3f247-9f1f-4136-961a-8b174c96f4d7, infoPort=46549, infoSecurePort=0, ipcPort=44203, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255) 2024-11-21T22:00:19,127 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc7c5dae8ed631b6f with lease ID 0xda84ec3abc2caf67: from storage DS-f3a73415-0859-4728-87c4-f86f9b8f7ae2 node DatanodeRegistration(127.0.0.1:43557, datanodeUuid=95d3f247-9f1f-4136-961a-8b174c96f4d7, infoPort=46549, infoSecurePort=0, ipcPort=44203, storageInfo=lv=-57;cid=testClusterID;nsid=1927353710;c=1732226416255), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T22:00:19,179 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8 2024-11-21T22:00:19,254 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f/zookeeper_0, clientPort=62211, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dc07c5e-8f73-378b-5a96-c766d8e86da8/cluster_c50adb9e-3b7c-f87e-639e-1a0548b3ad9f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T22:00:19,264 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62211 2024-11-21T22:00:19,277 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:19,280 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:19,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741825_1001 (size=7) 2024-11-21T22:00:19,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741825_1001 (size=7) 2024-11-21T22:00:19,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741825_1001 (size=7) 2024-11-21T22:00:19,909 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 with version=8 2024-11-21T22:00:19,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/hbase-staging 2024-11-21T22:00:19,984 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T22:00:20,211 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2a884b411602:0 server-side Connection retries=45 2024-11-21T22:00:20,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,220 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,224 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T22:00:20,224 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,224 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T22:00:20,351 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T22:00:20,403 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T22:00:20,411 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T22:00:20,414 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T22:00:20,439 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 104451 (auto-detected) 2024-11-21T22:00:20,440 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T22:00:20,461 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38029 2024-11-21T22:00:20,484 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38029 connecting to ZooKeeper ensemble=127.0.0.1:62211 2024-11-21T22:00:20,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:380290x0, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T22:00:20,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38029-0x10160b31ce40000 connected 2024-11-21T22:00:20,669 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:20,672 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:20,681 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T22:00:20,685 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419, hbase.cluster.distributed=false 2024-11-21T22:00:20,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T22:00:20,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38029 2024-11-21T22:00:20,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38029 2024-11-21T22:00:20,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38029 2024-11-21T22:00:20,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38029 2024-11-21T22:00:20,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38029 2024-11-21T22:00:20,824 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2a884b411602:0 server-side Connection retries=45 2024-11-21T22:00:20,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,826 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,826 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T22:00:20,826 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,826 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T22:00:20,829 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T22:00:20,832 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T22:00:20,835 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36681 2024-11-21T22:00:20,838 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36681 connecting to ZooKeeper ensemble=127.0.0.1:62211 2024-11-21T22:00:20,839 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:20,845 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:20,867 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366810x0, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T22:00:20,868 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T22:00:20,868 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36681-0x10160b31ce40001 connected 2024-11-21T22:00:20,872 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T22:00:20,881 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T22:00:20,884 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T22:00:20,890 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T22:00:20,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36681 2024-11-21T22:00:20,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36681 2024-11-21T22:00:20,895 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36681 2024-11-21T22:00:20,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36681 2024-11-21T22:00:20,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36681 2024-11-21T22:00:20,917 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2a884b411602:0 server-side Connection retries=45 2024-11-21T22:00:20,917 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,917 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,918 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T22:00:20,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T22:00:20,918 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T22:00:20,919 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T22:00:20,919 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34991 2024-11-21T22:00:20,921 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34991 connecting to ZooKeeper ensemble=127.0.0.1:62211 2024-11-21T22:00:20,922 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:20,926 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:20,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:349910x0, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T22:00:20,960 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:349910x0, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T22:00:20,960 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34991-0x10160b31ce40002 connected 2024-11-21T22:00:20,961 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T22:00:20,962 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T22:00:20,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T22:00:20,966 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T22:00:20,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34991 2024-11-21T22:00:20,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34991 2024-11-21T22:00:20,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34991 2024-11-21T22:00:20,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34991 2024-11-21T22:00:20,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34991 2024-11-21T22:00:20,989 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2a884b411602:0 server-side Connection retries=45 2024-11-21T22:00:20,989 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,989 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,989 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T22:00:20,989 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T22:00:20,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T22:00:20,990 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T22:00:20,990 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T22:00:20,991 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45209 2024-11-21T22:00:20,992 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45209 connecting to ZooKeeper ensemble=127.0.0.1:62211 2024-11-21T22:00:20,994 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:20,996 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:21,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452090x0, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T22:00:21,010 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452090x0, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T22:00:21,010 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45209-0x10160b31ce40003 connected 2024-11-21T22:00:21,010 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T22:00:21,014 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T22:00:21,016 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T22:00:21,017 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T22:00:21,018 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45209 2024-11-21T22:00:21,019 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45209 2024-11-21T22:00:21,019 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45209 2024-11-21T22:00:21,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45209 2024-11-21T22:00:21,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45209 2024-11-21T22:00:21,038 DEBUG [M:0;2a884b411602:38029 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2a884b411602:38029 2024-11-21T22:00:21,040 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2a884b411602,38029,1732226420059 2024-11-21T22:00:21,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T22:00:21,056 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T22:00:21,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T22:00:21,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T22:00:21,058 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2a884b411602,38029,1732226420059 2024-11-21T22:00:21,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T22:00:21,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T22:00:21,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,088 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T22:00:21,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,089 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,090 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T22:00:21,091 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2a884b411602,38029,1732226420059 from backup master directory 2024-11-21T22:00:21,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T22:00:21,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2a884b411602,38029,1732226420059 2024-11-21T22:00:21,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T22:00:21,101 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T22:00:21,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T22:00:21,102 WARN [master/2a884b411602:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T22:00:21,102 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2a884b411602,38029,1732226420059 2024-11-21T22:00:21,105 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T22:00:21,107 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T22:00:21,163 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/hbase.id] with ID: 70ad5833-ee99-4eba-8991-80a4feccbd08 2024-11-21T22:00:21,163 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/.tmp/hbase.id 2024-11-21T22:00:21,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741826_1002 (size=42) 2024-11-21T22:00:21,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741826_1002 (size=42) 2024-11-21T22:00:21,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741826_1002 (size=42) 2024-11-21T22:00:21,178 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/.tmp/hbase.id]:[hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/hbase.id] 2024-11-21T22:00:21,220 INFO [master/2a884b411602:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T22:00:21,224 INFO [master/2a884b411602:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T22:00:21,241 INFO [master/2a884b411602:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-11-21T22:00:21,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,264 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741827_1003 (size=196) 2024-11-21T22:00:21,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741827_1003 (size=196) 2024-11-21T22:00:21,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741827_1003 (size=196) 2024-11-21T22:00:21,303 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T22:00:21,305 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T22:00:21,317 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T22:00:21,320 INFO [master/2a884b411602:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T22:00:21,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741828_1004 (size=1189) 2024-11-21T22:00:21,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741828_1004 (size=1189) 2024-11-21T22:00:21,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741828_1004 (size=1189) 2024-11-21T22:00:21,364 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/data/master/store 2024-11-21T22:00:21,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741829_1005 (size=34) 2024-11-21T22:00:21,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741829_1005 (size=34) 2024-11-21T22:00:21,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741829_1005 (size=34) 2024-11-21T22:00:21,390 INFO [master/2a884b411602:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T22:00:21,393 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:21,394 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T22:00:21,394 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T22:00:21,394 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T22:00:21,395 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T22:00:21,395 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T22:00:21,396 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T22:00:21,397 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732226421394Disabling compacts and flushes for region at 1732226421394Disabling writes for close at 1732226421395 (+1 ms)Writing region close event to WAL at 1732226421396 (+1 ms)Closed at 1732226421396 2024-11-21T22:00:21,398 WARN [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/data/master/store/.initializing 2024-11-21T22:00:21,399 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/WALs/2a884b411602,38029,1732226420059 2024-11-21T22:00:21,406 INFO [master/2a884b411602:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T22:00:21,420 INFO [master/2a884b411602:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2a884b411602%2C38029%2C1732226420059, suffix=, logDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/WALs/2a884b411602,38029,1732226420059, archiveDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/oldWALs, maxLogs=10 2024-11-21T22:00:21,445 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/WALs/2a884b411602,38029,1732226420059/2a884b411602%2C38029%2C1732226420059.1732226421424, exclude list is [], retry=0 2024-11-21T22:00:21,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37045,DS-c564ed92-b78e-46e6-b4be-1100870e1511,DISK] 2024-11-21T22:00:21,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34171,DS-e91666bc-b006-4fed-88f2-dd54fc70cc22,DISK] 2024-11-21T22:00:21,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43557,DS-d8a9d7a5-362c-4f09-b1ac-e85456762592,DISK] 2024-11-21T22:00:21,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T22:00:21,497 INFO [master/2a884b411602:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/WALs/2a884b411602,38029,1732226420059/2a884b411602%2C38029%2C1732226420059.1732226421424 2024-11-21T22:00:21,498 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:43385:43385),(127.0.0.1/127.0.0.1:46549:46549)] 2024-11-21T22:00:21,498 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T22:00:21,499 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:21,502 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,503 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,559 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T22:00:21,562 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:21,564 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:21,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,567 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T22:00:21,568 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:21,568 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:21,569 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,571 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T22:00:21,571 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:21,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:21,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,575 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T22:00:21,575 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:21,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:21,576 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,580 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,581 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,587 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,587 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,591 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T22:00:21,595 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T22:00:21,599 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:21,601 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74477446, jitterRate=0.10980042815208435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T22:00:21,608 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732226421515Initializing all the Stores at 1732226421517 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226421518 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226421518Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226421518Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226421519 (+1 ms)Cleaning up temporary data from old regions at 1732226421588 (+69 ms)Region opened successfully at 1732226421608 (+20 ms) 2024-11-21T22:00:21,609 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T22:00:21,640 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59748d31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T22:00:21,667 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T22:00:21,677 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:21,677 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:21,679 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:21,681 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T22:00:21,687 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T22:00:21,687 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T22:00:21,710 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T22:00:21,718 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T22:00:21,750 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T22:00:21,753 INFO [master/2a884b411602:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T22:00:21,754 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T22:00:21,763 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T22:00:21,765 INFO [master/2a884b411602:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T22:00:21,769 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T22:00:21,780 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T22:00:21,781 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T22:00:21,792 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T22:00:21,808 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T22:00:21,817 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T22:00:21,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T22:00:21,830 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T22:00:21,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T22:00:21,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T22:00:21,830 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,833 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2a884b411602,38029,1732226420059, sessionid=0x10160b31ce40000, setting cluster-up flag (Was=false) 2024-11-21T22:00:21,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,867 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,892 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T22:00:21,894 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2a884b411602,38029,1732226420059 2024-11-21T22:00:21,913 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:21,942 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T22:00:21,944 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2a884b411602,38029,1732226420059 2024-11-21T22:00:21,951 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T22:00:22,019 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:22,026 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(746): ClusterId : 70ad5833-ee99-4eba-8991-80a4feccbd08 2024-11-21T22:00:22,026 INFO [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer(746): ClusterId : 70ad5833-ee99-4eba-8991-80a4feccbd08 2024-11-21T22:00:22,026 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(746): ClusterId : 70ad5833-ee99-4eba-8991-80a4feccbd08 2024-11-21T22:00:22,028 DEBUG [RS:2;2a884b411602:45209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T22:00:22,028 DEBUG [RS:0;2a884b411602:36681 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T22:00:22,028 DEBUG [RS:1;2a884b411602:34991 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T22:00:22,029 INFO [master/2a884b411602:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T22:00:22,035 INFO [master/2a884b411602:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T22:00:22,042 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2a884b411602,38029,1732226420059 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T22:00:22,057 DEBUG [RS:0;2a884b411602:36681 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T22:00:22,057 DEBUG [RS:2;2a884b411602:45209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T22:00:22,057 DEBUG [RS:1;2a884b411602:34991 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T22:00:22,057 DEBUG [RS:0;2a884b411602:36681 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T22:00:22,057 DEBUG [RS:2;2a884b411602:45209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T22:00:22,057 DEBUG [RS:1;2a884b411602:34991 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T22:00:22,058 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T22:00:22,058 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T22:00:22,058 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T22:00:22,058 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T22:00:22,058 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2a884b411602:0, corePoolSize=10, maxPoolSize=10 2024-11-21T22:00:22,058 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,059 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T22:00:22,059 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,059 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226452059 2024-11-21T22:00:22,061 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T22:00:22,062 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T22:00:22,064 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:22,065 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T22:00:22,066 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T22:00:22,067 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T22:00:22,067 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T22:00:22,067 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T22:00:22,068 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,070 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T22:00:22,071 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:22,071 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T22:00:22,072 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T22:00:22,071 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T22:00:22,073 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T22:00:22,074 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T22:00:22,076 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.large.0-1732226422075,5,FailOnTimeoutGroup] 2024-11-21T22:00:22,076 DEBUG [RS:1;2a884b411602:34991 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T22:00:22,076 DEBUG [RS:0;2a884b411602:36681 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T22:00:22,076 DEBUG [RS:2;2a884b411602:45209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T22:00:22,077 DEBUG [RS:1;2a884b411602:34991 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57cdaa34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T22:00:22,077 DEBUG [RS:0;2a884b411602:36681 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1d248d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T22:00:22,077 DEBUG [RS:2;2a884b411602:45209 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de5e699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T22:00:22,078 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.small.0-1732226422076,5,FailOnTimeoutGroup] 2024-11-21T22:00:22,079 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,079 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T22:00:22,080 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,080 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,097 DEBUG [RS:2;2a884b411602:45209 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2a884b411602:45209 2024-11-21T22:00:22,097 DEBUG [RS:0;2a884b411602:36681 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2a884b411602:36681 2024-11-21T22:00:22,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741831_1007 (size=1321) 2024-11-21T22:00:22,099 DEBUG [RS:1;2a884b411602:34991 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2a884b411602:34991 2024-11-21T22:00:22,101 INFO [RS:1;2a884b411602:34991 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T22:00:22,101 INFO [RS:2;2a884b411602:45209 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T22:00:22,101 INFO [RS:1;2a884b411602:34991 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T22:00:22,101 INFO [RS:2;2a884b411602:45209 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T22:00:22,101 DEBUG [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T22:00:22,101 DEBUG [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T22:00:22,102 INFO [RS:0;2a884b411602:36681 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T22:00:22,102 INFO [RS:0;2a884b411602:36681 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T22:00:22,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741831_1007 (size=1321) 2024-11-21T22:00:22,102 DEBUG [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T22:00:22,104 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T22:00:22,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741831_1007 (size=1321) 2024-11-21T22:00:22,104 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:22,104 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(2659): reportForDuty to master=2a884b411602,38029,1732226420059 with port=45209, startcode=1732226420988 2024-11-21T22:00:22,105 INFO [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer(2659): reportForDuty to master=2a884b411602,38029,1732226420059 with port=34991, startcode=1732226420916 2024-11-21T22:00:22,105 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(2659): reportForDuty to master=2a884b411602,38029,1732226420059 with port=36681, startcode=1732226420789 2024-11-21T22:00:22,119 DEBUG [RS:2;2a884b411602:45209 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T22:00:22,119 DEBUG [RS:1;2a884b411602:34991 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T22:00:22,119 DEBUG [RS:0;2a884b411602:36681 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T22:00:22,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741832_1008 (size=32) 2024-11-21T22:00:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741832_1008 (size=32) 2024-11-21T22:00:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741832_1008 (size=32) 2024-11-21T22:00:22,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:22,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T22:00:22,166 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T22:00:22,166 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:22,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:22,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T22:00:22,177 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T22:00:22,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:22,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:22,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T22:00:22,188 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T22:00:22,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:22,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:22,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T22:00:22,193 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60605, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T22:00:22,193 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33219, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T22:00:22,193 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T22:00:22,193 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:22,195 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49461, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T22:00:22,200 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2a884b411602,34991,1732226420916 2024-11-21T22:00:22,203 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] master.ServerManager(517): Registering regionserver=2a884b411602,34991,1732226420916 2024-11-21T22:00:22,204 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:22,205 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T22:00:22,209 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740 2024-11-21T22:00:22,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740 2024-11-21T22:00:22,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T22:00:22,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T22:00:22,218 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2a884b411602,36681,1732226420789 2024-11-21T22:00:22,218 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] master.ServerManager(517): Registering regionserver=2a884b411602,36681,1732226420789 2024-11-21T22:00:22,219 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T22:00:22,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T22:00:22,225 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2a884b411602,45209,1732226420988 2024-11-21T22:00:22,225 DEBUG [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:22,225 DEBUG [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38269 2024-11-21T22:00:22,225 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] master.ServerManager(517): Registering regionserver=2a884b411602,45209,1732226420988 2024-11-21T22:00:22,225 DEBUG [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T22:00:22,228 DEBUG [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:22,228 DEBUG [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38269 2024-11-21T22:00:22,228 DEBUG [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T22:00:22,231 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:22,231 DEBUG [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:22,231 DEBUG [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38269 2024-11-21T22:00:22,231 DEBUG [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T22:00:22,232 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72959492, jitterRate=0.08718115091323853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T22:00:22,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732226422159Initializing all the Stores at 1732226422161 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226422161Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226422162 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226422162Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226422163 (+1 ms)Cleaning up temporary data from old regions at 1732226422216 (+53 ms)Region opened successfully at 1732226422236 (+20 ms) 2024-11-21T22:00:22,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T22:00:22,237 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T22:00:22,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T22:00:22,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T22:00:22,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T22:00:22,239 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T22:00:22,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732226422236Disabling compacts and flushes for region at 1732226422236Disabling writes for close at 1732226422237 (+1 ms)Writing region close event to WAL at 1732226422239 (+2 ms)Closed at 1732226422239 2024-11-21T22:00:22,242 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:22,242 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T22:00:22,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T22:00:22,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T22:00:22,259 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:22,262 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T22:00:22,297 DEBUG [RS:1;2a884b411602:34991 {}] zookeeper.ZKUtil(111): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2a884b411602,34991,1732226420916 2024-11-21T22:00:22,297 WARN [RS:1;2a884b411602:34991 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T22:00:22,297 DEBUG [RS:2;2a884b411602:45209 {}] zookeeper.ZKUtil(111): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2a884b411602,45209,1732226420988 2024-11-21T22:00:22,297 WARN [RS:2;2a884b411602:45209 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T22:00:22,297 INFO [RS:1;2a884b411602:34991 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T22:00:22,297 INFO [RS:2;2a884b411602:45209 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T22:00:22,297 DEBUG [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916 2024-11-21T22:00:22,297 DEBUG [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,45209,1732226420988 2024-11-21T22:00:22,298 DEBUG [RS:0;2a884b411602:36681 {}] zookeeper.ZKUtil(111): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2a884b411602,36681,1732226420789 2024-11-21T22:00:22,298 WARN [RS:0;2a884b411602:36681 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T22:00:22,298 INFO [RS:0;2a884b411602:36681 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T22:00:22,298 DEBUG [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,36681,1732226420789 2024-11-21T22:00:22,298 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2a884b411602,36681,1732226420789] 2024-11-21T22:00:22,299 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2a884b411602,45209,1732226420988] 2024-11-21T22:00:22,299 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2a884b411602,34991,1732226420916] 2024-11-21T22:00:22,328 INFO [RS:2;2a884b411602:45209 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T22:00:22,328 INFO [RS:1;2a884b411602:34991 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T22:00:22,328 INFO [RS:0;2a884b411602:36681 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T22:00:22,344 INFO [RS:1;2a884b411602:34991 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T22:00:22,345 INFO [RS:0;2a884b411602:36681 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T22:00:22,345 INFO [RS:2;2a884b411602:45209 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T22:00:22,350 INFO [RS:1;2a884b411602:34991 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T22:00:22,350 INFO [RS:2;2a884b411602:45209 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T22:00:22,350 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,350 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,352 INFO [RS:0;2a884b411602:36681 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T22:00:22,352 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,352 INFO [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T22:00:22,355 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T22:00:22,359 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T22:00:22,360 INFO [RS:1;2a884b411602:34991 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T22:00:22,360 INFO [RS:2;2a884b411602:45209 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T22:00:22,360 INFO [RS:0;2a884b411602:36681 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T22:00:22,361 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,361 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,361 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,362 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,362 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,362 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,362 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,362 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,362 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,362 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T22:00:22,363 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T22:00:22,363 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,363 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,364 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,364 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,364 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,364 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,364 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,364 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T22:00:22,364 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T22:00:22,364 DEBUG [RS:1;2a884b411602:34991 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T22:00:22,364 DEBUG [RS:0;2a884b411602:36681 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T22:00:22,364 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,365 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,365 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,365 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,365 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,365 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T22:00:22,365 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,365 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,366 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,366 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,366 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,366 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T22:00:22,366 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T22:00:22,367 DEBUG [RS:2;2a884b411602:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T22:00:22,367 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,367 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,367 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,367 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,367 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,367 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,34991,1732226420916-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T22:00:22,374 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,374 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,374 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,374 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,374 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,374 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,375 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,375 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,375 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,36681,1732226420789-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T22:00:22,375 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,375 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,375 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,45209,1732226420988-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T22:00:22,396 INFO [RS:2;2a884b411602:45209 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T22:00:22,396 INFO [RS:1;2a884b411602:34991 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T22:00:22,397 INFO [RS:0;2a884b411602:36681 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T22:00:22,398 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,45209,1732226420988-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,398 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,34991,1732226420916-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,398 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,36681,1732226420789-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,399 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,399 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,399 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,399 INFO [RS:0;2a884b411602:36681 {}] regionserver.Replication(171): 2a884b411602,36681,1732226420789 started 2024-11-21T22:00:22,399 INFO [RS:2;2a884b411602:45209 {}] regionserver.Replication(171): 2a884b411602,45209,1732226420988 started 2024-11-21T22:00:22,399 INFO [RS:1;2a884b411602:34991 {}] regionserver.Replication(171): 2a884b411602,34991,1732226420916 started 2024-11-21T22:00:22,413 WARN [2a884b411602:38029 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T22:00:22,419 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,419 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(1482): Serving as 2a884b411602,45209,1732226420988, RpcServer on 2a884b411602/172.17.0.2:45209, sessionid=0x10160b31ce40003 2024-11-21T22:00:22,420 DEBUG [RS:2;2a884b411602:45209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T22:00:22,420 DEBUG [RS:2;2a884b411602:45209 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2a884b411602,45209,1732226420988 2024-11-21T22:00:22,420 DEBUG [RS:2;2a884b411602:45209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,45209,1732226420988' 2024-11-21T22:00:22,420 DEBUG [RS:2;2a884b411602:45209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T22:00:22,421 DEBUG [RS:2;2a884b411602:45209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T22:00:22,422 DEBUG [RS:2;2a884b411602:45209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T22:00:22,422 DEBUG [RS:2;2a884b411602:45209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T22:00:22,422 DEBUG [RS:2;2a884b411602:45209 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2a884b411602,45209,1732226420988 2024-11-21T22:00:22,422 DEBUG [RS:2;2a884b411602:45209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,45209,1732226420988' 2024-11-21T22:00:22,422 DEBUG [RS:2;2a884b411602:45209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T22:00:22,423 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,423 INFO [RS:1;2a884b411602:34991 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:22,423 DEBUG [RS:2;2a884b411602:45209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T22:00:22,423 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(1482): Serving as 2a884b411602,36681,1732226420789, RpcServer on 2a884b411602/172.17.0.2:36681, sessionid=0x10160b31ce40001 2024-11-21T22:00:22,423 INFO [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer(1482): Serving as 2a884b411602,34991,1732226420916, RpcServer on 2a884b411602/172.17.0.2:34991, sessionid=0x10160b31ce40002 2024-11-21T22:00:22,423 DEBUG [RS:0;2a884b411602:36681 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T22:00:22,423 DEBUG [RS:1;2a884b411602:34991 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T22:00:22,423 DEBUG [RS:0;2a884b411602:36681 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2a884b411602,36681,1732226420789 2024-11-21T22:00:22,423 DEBUG [RS:1;2a884b411602:34991 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2a884b411602,34991,1732226420916 2024-11-21T22:00:22,423 DEBUG [RS:0;2a884b411602:36681 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,36681,1732226420789' 2024-11-21T22:00:22,423 DEBUG [RS:1;2a884b411602:34991 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,34991,1732226420916' 2024-11-21T22:00:22,423 DEBUG [RS:0;2a884b411602:36681 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T22:00:22,423 DEBUG [RS:1;2a884b411602:34991 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T22:00:22,423 DEBUG [RS:2;2a884b411602:45209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T22:00:22,424 INFO [RS:2;2a884b411602:45209 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T22:00:22,424 INFO [RS:2;2a884b411602:45209 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T22:00:22,424 DEBUG [RS:1;2a884b411602:34991 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T22:00:22,424 DEBUG [RS:0;2a884b411602:36681 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T22:00:22,424 DEBUG [RS:1;2a884b411602:34991 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T22:00:22,424 DEBUG [RS:0;2a884b411602:36681 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T22:00:22,425 DEBUG [RS:1;2a884b411602:34991 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T22:00:22,425 DEBUG [RS:0;2a884b411602:36681 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T22:00:22,425 DEBUG [RS:1;2a884b411602:34991 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2a884b411602,34991,1732226420916 2024-11-21T22:00:22,425 DEBUG [RS:1;2a884b411602:34991 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,34991,1732226420916' 2024-11-21T22:00:22,425 DEBUG [RS:0;2a884b411602:36681 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2a884b411602,36681,1732226420789 2024-11-21T22:00:22,425 DEBUG [RS:1;2a884b411602:34991 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T22:00:22,425 DEBUG [RS:0;2a884b411602:36681 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,36681,1732226420789' 2024-11-21T22:00:22,425 DEBUG [RS:0;2a884b411602:36681 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T22:00:22,425 DEBUG [RS:1;2a884b411602:34991 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T22:00:22,425 DEBUG [RS:0;2a884b411602:36681 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T22:00:22,426 DEBUG [RS:1;2a884b411602:34991 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T22:00:22,426 INFO [RS:1;2a884b411602:34991 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T22:00:22,426 DEBUG [RS:0;2a884b411602:36681 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T22:00:22,426 INFO [RS:1;2a884b411602:34991 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T22:00:22,426 INFO [RS:0;2a884b411602:36681 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T22:00:22,426 INFO [RS:0;2a884b411602:36681 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T22:00:22,532 INFO [RS:2;2a884b411602:45209 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T22:00:22,532 INFO [RS:1;2a884b411602:34991 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T22:00:22,532 INFO [RS:0;2a884b411602:36681 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T22:00:22,537 INFO [RS:1;2a884b411602:34991 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2a884b411602%2C34991%2C1732226420916, suffix=, logDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916, archiveDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/oldWALs, maxLogs=32 2024-11-21T22:00:22,537 INFO [RS:2;2a884b411602:45209 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2a884b411602%2C45209%2C1732226420988, suffix=, logDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,45209,1732226420988, archiveDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/oldWALs, maxLogs=32 2024-11-21T22:00:22,537 INFO [RS:0;2a884b411602:36681 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2a884b411602%2C36681%2C1732226420789, suffix=, logDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,36681,1732226420789, archiveDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/oldWALs, maxLogs=32 2024-11-21T22:00:22,554 DEBUG [RS:2;2a884b411602:45209 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,45209,1732226420988/2a884b411602%2C45209%2C1732226420988.1732226422541, exclude list is [], retry=0 2024-11-21T22:00:22,554 DEBUG [RS:1;2a884b411602:34991 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916/2a884b411602%2C34991%2C1732226420916.1732226422541, exclude list is [], retry=0 2024-11-21T22:00:22,555 DEBUG [RS:0;2a884b411602:36681 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,36681,1732226420789/2a884b411602%2C36681%2C1732226420789.1732226422542, exclude list is [], retry=0 2024-11-21T22:00:22,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34171,DS-e91666bc-b006-4fed-88f2-dd54fc70cc22,DISK] 2024-11-21T22:00:22,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43557,DS-d8a9d7a5-362c-4f09-b1ac-e85456762592,DISK] 2024-11-21T22:00:22,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34171,DS-e91666bc-b006-4fed-88f2-dd54fc70cc22,DISK] 2024-11-21T22:00:22,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37045,DS-c564ed92-b78e-46e6-b4be-1100870e1511,DISK] 2024-11-21T22:00:22,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37045,DS-c564ed92-b78e-46e6-b4be-1100870e1511,DISK] 2024-11-21T22:00:22,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37045,DS-c564ed92-b78e-46e6-b4be-1100870e1511,DISK] 2024-11-21T22:00:22,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34171,DS-e91666bc-b006-4fed-88f2-dd54fc70cc22,DISK] 2024-11-21T22:00:22,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43557,DS-d8a9d7a5-362c-4f09-b1ac-e85456762592,DISK] 2024-11-21T22:00:22,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43557,DS-d8a9d7a5-362c-4f09-b1ac-e85456762592,DISK] 2024-11-21T22:00:22,568 INFO [RS:1;2a884b411602:34991 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916/2a884b411602%2C34991%2C1732226420916.1732226422541 2024-11-21T22:00:22,572 INFO [RS:2;2a884b411602:45209 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,45209,1732226420988/2a884b411602%2C45209%2C1732226420988.1732226422541 2024-11-21T22:00:22,573 DEBUG [RS:1;2a884b411602:34991 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:43385:43385),(127.0.0.1/127.0.0.1:46549:46549)] 2024-11-21T22:00:22,573 DEBUG [RS:2;2a884b411602:45209 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46549:46549),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:43385:43385)] 2024-11-21T22:00:22,574 INFO [RS:0;2a884b411602:36681 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,36681,1732226420789/2a884b411602%2C36681%2C1732226420789.1732226422542 2024-11-21T22:00:22,574 DEBUG [RS:0;2a884b411602:36681 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43385:43385),(127.0.0.1/127.0.0.1:46549:46549),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-21T22:00:22,669 DEBUG [2a884b411602:38029 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-21T22:00:22,679 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T22:00:22,685 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T22:00:22,685 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T22:00:22,685 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T22:00:22,685 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T22:00:22,685 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T22:00:22,685 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T22:00:22,685 INFO [2a884b411602:38029 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T22:00:22,685 INFO [2a884b411602:38029 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T22:00:22,685 INFO [2a884b411602:38029 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T22:00:22,685 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T22:00:22,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:22,700 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,34991,1732226420916, state=OPENING 2024-11-21T22:00:22,730 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T22:00:22,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:22,780 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:22,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:22,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:22,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:22,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:22,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:22,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:22,782 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:22,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:22,961 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T22:00:22,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50841, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T22:00:22,975 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T22:00:22,976 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T22:00:22,976 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T22:00:22,980 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2a884b411602%2C34991%2C1732226420916.meta, suffix=.meta, logDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916, archiveDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/oldWALs, maxLogs=32 2024-11-21T22:00:22,996 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, exclude list is [], retry=0 2024-11-21T22:00:23,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43557,DS-d8a9d7a5-362c-4f09-b1ac-e85456762592,DISK] 2024-11-21T22:00:23,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37045,DS-c564ed92-b78e-46e6-b4be-1100870e1511,DISK] 2024-11-21T22:00:23,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34171,DS-e91666bc-b006-4fed-88f2-dd54fc70cc22,DISK] 2024-11-21T22:00:23,003 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta 2024-11-21T22:00:23,003 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43385:43385),(127.0.0.1/127.0.0.1:46549:46549),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-21T22:00:23,003 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T22:00:23,005 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T22:00:23,007 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T22:00:23,012 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T22:00:23,017 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T22:00:23,018 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,018 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T22:00:23,018 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T22:00:23,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T22:00:23,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T22:00:23,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:23,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:23,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T22:00:23,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T22:00:23,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:23,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:23,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T22:00:23,028 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T22:00:23,028 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:23,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:23,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T22:00:23,030 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T22:00:23,030 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:23,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:23,031 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T22:00:23,033 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740 2024-11-21T22:00:23,035 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740 2024-11-21T22:00:23,037 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T22:00:23,038 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T22:00:23,038 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T22:00:23,041 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T22:00:23,043 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62434296, jitterRate=-0.06965649127960205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T22:00:23,043 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T22:00:23,044 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732226423018Writing region info on filesystem at 1732226423019 (+1 ms)Initializing all the Stores at 1732226423020 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226423020Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226423021 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226423021Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226423021Cleaning up temporary data from old regions at 1732226423038 (+17 ms)Running coprocessor post-open hooks at 1732226423043 (+5 ms)Region opened successfully at 1732226423044 (+1 ms) 2024-11-21T22:00:23,050 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732226422949 2024-11-21T22:00:23,060 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T22:00:23,061 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T22:00:23,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:23,064 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,34991,1732226420916, state=OPEN 2024-11-21T22:00:23,096 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:23,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:23,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:23,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:23,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:23,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:23,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:23,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:23,097 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2a884b411602,34991,1732226420916 2024-11-21T22:00:23,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T22:00:23,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,34991,1732226420916 in 314 msec 2024-11-21T22:00:23,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T22:00:23,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 857 msec 2024-11-21T22:00:23,114 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:23,114 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T22:00:23,132 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T22:00:23,133 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1] 2024-11-21T22:00:23,151 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T22:00:23,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46153, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T22:00:23,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1910 sec 2024-11-21T22:00:23,175 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732226423175, completionTime=-1 2024-11-21T22:00:23,177 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-21T22:00:23,177 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:23,201 INFO [master/2a884b411602:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-21T22:00:23,201 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226483201 2024-11-21T22:00:23,201 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226543201 2024-11-21T22:00:23,201 INFO [master/2a884b411602:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-21T22:00:23,202 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:23,208 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,38029,1732226420059-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:23,209 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,38029,1732226420059-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:23,209 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,38029,1732226420059-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:23,210 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2a884b411602:38029, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:23,210 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:23,211 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:23,221 DEBUG [master/2a884b411602:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T22:00:23,242 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.138sec 2024-11-21T22:00:23,243 INFO [master/2a884b411602:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T22:00:23,244 INFO [master/2a884b411602:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T22:00:23,245 INFO [master/2a884b411602:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T22:00:23,246 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T22:00:23,246 INFO [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T22:00:23,247 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,38029,1732226420059-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T22:00:23,248 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,38029,1732226420059-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T22:00:23,252 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T22:00:23,254 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T22:00:23,254 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,38029,1732226420059-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T22:00:23,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f0836e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T22:00:23,340 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T22:00:23,340 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T22:00:23,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2a884b411602,38029,-1 for getting cluster id 2024-11-21T22:00:23,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T22:00:23,352 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '70ad5833-ee99-4eba-8991-80a4feccbd08' 2024-11-21T22:00:23,354 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T22:00:23,354 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "70ad5833-ee99-4eba-8991-80a4feccbd08" 2024-11-21T22:00:23,354 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d908a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T22:00:23,354 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2a884b411602,38029,-1] 2024-11-21T22:00:23,356 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T22:00:23,359 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:23,359 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T22:00:23,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41704ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T22:00:23,363 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T22:00:23,369 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1] 2024-11-21T22:00:23,369 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T22:00:23,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T22:00:23,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2a884b411602,38029,1732226420059 2024-11-21T22:00:23,393 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T22:00:23,396 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2a884b411602,38029,1732226420059 2024-11-21T22:00:23,399 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7848b6a2 2024-11-21T22:00:23,399 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T22:00:23,402 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T22:00:23,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T22:00:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:23,438 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T22:00:23,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-21T22:00:23,440 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:23,443 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T22:00:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T22:00:23,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741837_1013 (size=358) 2024-11-21T22:00:23,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741837_1013 (size=358) 2024-11-21T22:00:23,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741837_1013 (size=358) 2024-11-21T22:00:23,479 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e8aafe78c77f56bf1d0a5753056c1486, NAME => 'test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,479 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f98ec44fefd547dcab3f48a26fc1df58, NAME => 'test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,479 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5737853cedadc75f4ba670c884671cc8, NAME => 'test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741838_1014 (size=45) 2024-11-21T22:00:23,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741838_1014 (size=45) 2024-11-21T22:00:23,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741838_1014 (size=45) 2024-11-21T22:00:23,499 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,499 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e8aafe78c77f56bf1d0a5753056c1486, disabling compactions & flushes 2024-11-21T22:00:23,499 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. 2024-11-21T22:00:23,499 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. 2024-11-21T22:00:23,499 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. after waiting 0 ms 2024-11-21T22:00:23,499 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. 2024-11-21T22:00:23,499 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. 2024-11-21T22:00:23,500 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e8aafe78c77f56bf1d0a5753056c1486: Waiting for close lock at 1732226423499Disabling compacts and flushes for region at 1732226423499Disabling writes for close at 1732226423499Writing region close event to WAL at 1732226423499Closed at 1732226423499 2024-11-21T22:00:23,500 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d6cc8ee4cfeffaf8625f8fdd19f2e2ff, NAME => 'test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741839_1015 (size=45) 2024-11-21T22:00:23,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741839_1015 (size=45) 2024-11-21T22:00:23,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741839_1015 (size=45) 2024-11-21T22:00:23,502 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,503 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5737853cedadc75f4ba670c884671cc8, disabling compactions & flushes 2024-11-21T22:00:23,503 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:23,503 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:23,503 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. after waiting 0 ms 2024-11-21T22:00:23,503 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:23,503 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:23,503 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5737853cedadc75f4ba670c884671cc8: Waiting for close lock at 1732226423503Disabling compacts and flushes for region at 1732226423503Disabling writes for close at 1732226423503Writing region close event to WAL at 1732226423503Closed at 1732226423503 2024-11-21T22:00:23,504 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 37682a4b3efbf9497991cc6bf0d15002, NAME => 'test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741840_1016 (size=42) 2024-11-21T22:00:23,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741840_1016 (size=42) 2024-11-21T22:00:23,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741840_1016 (size=42) 2024-11-21T22:00:23,511 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,511 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f98ec44fefd547dcab3f48a26fc1df58, disabling compactions & flushes 2024-11-21T22:00:23,511 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:23,511 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:23,512 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. after waiting 0 ms 2024-11-21T22:00:23,512 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:23,512 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:23,512 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f98ec44fefd547dcab3f48a26fc1df58: Waiting for close lock at 1732226423511Disabling compacts and flushes for region at 1732226423511Disabling writes for close at 1732226423512 (+1 ms)Writing region close event to WAL at 1732226423512Closed at 1732226423512 2024-11-21T22:00:23,512 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 063f9af8e6526f98d7dbe9567a19314c, NAME => 'test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741842_1018 (size=45) 2024-11-21T22:00:23,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741842_1018 (size=45) 2024-11-21T22:00:23,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741842_1018 (size=45) 2024-11-21T22:00:23,526 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,527 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d6cc8ee4cfeffaf8625f8fdd19f2e2ff, disabling compactions & flushes 2024-11-21T22:00:23,527 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:23,527 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:23,527 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. after waiting 0 ms 2024-11-21T22:00:23,527 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:23,527 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:23,527 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d6cc8ee4cfeffaf8625f8fdd19f2e2ff: Waiting for close lock at 1732226423526Disabling compacts and flushes for region at 1732226423526Disabling writes for close at 1732226423527 (+1 ms)Writing region close event to WAL at 1732226423527Closed at 1732226423527 2024-11-21T22:00:23,528 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 30e7bcb84c1a622f86b573b1a579eabb, NAME => 'test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741841_1017 (size=45) 2024-11-21T22:00:23,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741841_1017 (size=45) 2024-11-21T22:00:23,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741841_1017 (size=45) 2024-11-21T22:00:23,534 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741843_1019 (size=45) 2024-11-21T22:00:23,535 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 37682a4b3efbf9497991cc6bf0d15002, disabling compactions & flushes 2024-11-21T22:00:23,535 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:23,535 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:23,535 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. after waiting 0 ms 2024-11-21T22:00:23,535 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:23,535 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:23,535 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 37682a4b3efbf9497991cc6bf0d15002: Waiting for close lock at 1732226423535Disabling compacts and flushes for region at 1732226423535Disabling writes for close at 1732226423535Writing region close event to WAL at 1732226423535Closed at 1732226423535 2024-11-21T22:00:23,536 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => cc4fa100e1a67451c5548a58b0b04f47, NAME => 'test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,537 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741843_1019 (size=45) 2024-11-21T22:00:23,538 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 063f9af8e6526f98d7dbe9567a19314c, disabling compactions & flushes 2024-11-21T22:00:23,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741843_1019 (size=45) 2024-11-21T22:00:23,538 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:23,538 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:23,538 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. after waiting 0 ms 2024-11-21T22:00:23,538 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:23,538 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:23,538 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 063f9af8e6526f98d7dbe9567a19314c: Waiting for close lock at 1732226423538Disabling compacts and flushes for region at 1732226423538Disabling writes for close at 1732226423538Writing region close event to WAL at 1732226423538Closed at 1732226423538 2024-11-21T22:00:23,539 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 46ca3887dfa777a1bf18ad3f42c5ddfb, NAME => 'test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741844_1020 (size=45) 2024-11-21T22:00:23,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741844_1020 (size=45) 2024-11-21T22:00:23,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741844_1020 (size=45) 2024-11-21T22:00:23,554 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,554 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 30e7bcb84c1a622f86b573b1a579eabb, disabling compactions & flushes 2024-11-21T22:00:23,554 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:23,554 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:23,555 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. after waiting 0 ms 2024-11-21T22:00:23,555 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:23,555 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:23,555 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 30e7bcb84c1a622f86b573b1a579eabb: Waiting for close lock at 1732226423554Disabling compacts and flushes for region at 1732226423554Disabling writes for close at 1732226423555 (+1 ms)Writing region close event to WAL at 1732226423555Closed at 1732226423555 2024-11-21T22:00:23,556 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 95832a57de44abb16e1581ee9057cab8, NAME => 'test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T22:00:23,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741845_1021 (size=45) 2024-11-21T22:00:23,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741845_1021 (size=45) 2024-11-21T22:00:23,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741845_1021 (size=45) 2024-11-21T22:00:23,569 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,569 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing cc4fa100e1a67451c5548a58b0b04f47, disabling compactions & flushes 2024-11-21T22:00:23,569 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:23,569 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:23,569 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. after waiting 0 ms 2024-11-21T22:00:23,569 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:23,569 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:23,570 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for cc4fa100e1a67451c5548a58b0b04f47: Waiting for close lock at 1732226423569Disabling compacts and flushes for region at 1732226423569Disabling writes for close at 1732226423569Writing region close event to WAL at 1732226423569Closed at 1732226423569 2024-11-21T22:00:23,570 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9ad08ad34efe19c87e38729ba3c5e49f, NAME => 'test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741846_1022 (size=45) 2024-11-21T22:00:23,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741846_1022 (size=45) 2024-11-21T22:00:23,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741846_1022 (size=45) 2024-11-21T22:00:23,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 46ca3887dfa777a1bf18ad3f42c5ddfb, disabling compactions & flushes 2024-11-21T22:00:23,579 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:23,579 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:23,579 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. after waiting 0 ms 2024-11-21T22:00:23,579 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:23,579 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:23,579 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 46ca3887dfa777a1bf18ad3f42c5ddfb: Waiting for close lock at 1732226423578Disabling compacts and flushes for region at 1732226423578Disabling writes for close at 1732226423579 (+1 ms)Writing region close event to WAL at 1732226423579Closed at 1732226423579 2024-11-21T22:00:23,580 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 679d18a37ac81fe200738b66f744f5cf, NAME => 'test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741847_1023 (size=45) 2024-11-21T22:00:23,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741847_1023 (size=45) 2024-11-21T22:00:23,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741847_1023 (size=45) 2024-11-21T22:00:23,588 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,588 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 95832a57de44abb16e1581ee9057cab8, disabling compactions & flushes 2024-11-21T22:00:23,588 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:23,588 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:23,588 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. after waiting 0 ms 2024-11-21T22:00:23,588 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:23,588 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:23,589 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 95832a57de44abb16e1581ee9057cab8: Waiting for close lock at 1732226423588Disabling compacts and flushes for region at 1732226423588Disabling writes for close at 1732226423588Writing region close event to WAL at 1732226423588Closed at 1732226423588 2024-11-21T22:00:23,589 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e151130b05e8b6cf9cbcab7d52f2339d, NAME => 'test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741848_1024 (size=45) 2024-11-21T22:00:23,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741848_1024 (size=45) 2024-11-21T22:00:23,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741848_1024 (size=45) 2024-11-21T22:00:23,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 9ad08ad34efe19c87e38729ba3c5e49f, disabling compactions & flushes 2024-11-21T22:00:23,599 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:23,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:23,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. after waiting 0 ms 2024-11-21T22:00:23,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:23,599 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:23,600 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9ad08ad34efe19c87e38729ba3c5e49f: Waiting for close lock at 1732226423599Disabling compacts and flushes for region at 1732226423599Disabling writes for close at 1732226423599Writing region close event to WAL at 1732226423599Closed at 1732226423599 2024-11-21T22:00:23,601 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bb120caa77e7b0578515651fde091e69, NAME => 'test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741849_1025 (size=45) 2024-11-21T22:00:23,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741849_1025 (size=45) 2024-11-21T22:00:23,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741849_1025 (size=45) 2024-11-21T22:00:23,606 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,606 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 679d18a37ac81fe200738b66f744f5cf, disabling compactions & flushes 2024-11-21T22:00:23,606 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:23,606 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:23,606 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. after waiting 0 ms 2024-11-21T22:00:23,606 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:23,606 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:23,606 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 679d18a37ac81fe200738b66f744f5cf: Waiting for close lock at 1732226423606Disabling compacts and flushes for region at 1732226423606Disabling writes for close at 1732226423606Writing region close event to WAL at 1732226423606Closed at 1732226423606 2024-11-21T22:00:23,607 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5115225b7a59d832f569746a9b524ee8, NAME => 'test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741850_1026 (size=45) 2024-11-21T22:00:23,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741851_1027 (size=45) 2024-11-21T22:00:23,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741850_1026 (size=45) 2024-11-21T22:00:23,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741851_1027 (size=45) 2024-11-21T22:00:23,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741851_1027 (size=45) 2024-11-21T22:00:23,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741850_1026 (size=45) 2024-11-21T22:00:23,618 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,618 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing bb120caa77e7b0578515651fde091e69, disabling compactions & flushes 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e151130b05e8b6cf9cbcab7d52f2339d, disabling compactions & flushes 2024-11-21T22:00:23,619 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. 2024-11-21T22:00:23,619 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. after waiting 0 ms 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. after waiting 0 ms 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:23,619 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. 2024-11-21T22:00:23,619 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for bb120caa77e7b0578515651fde091e69: Waiting for close lock at 1732226423619Disabling compacts and flushes for region at 1732226423619Disabling writes for close at 1732226423619Writing region close event to WAL at 1732226423619Closed at 1732226423619 2024-11-21T22:00:23,619 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e151130b05e8b6cf9cbcab7d52f2339d: Waiting for close lock at 1732226423619Disabling compacts and flushes for region at 1732226423619Disabling writes for close at 1732226423619Writing region close event to WAL at 1732226423619Closed at 1732226423619 2024-11-21T22:00:23,620 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d58deb362db49650ac0424f413c0d8a1, NAME => 'test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,620 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 17a273c548a9d111239f960bce54c80e, NAME => 'test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741852_1028 (size=45) 2024-11-21T22:00:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741852_1028 (size=45) 2024-11-21T22:00:23,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741852_1028 (size=45) 2024-11-21T22:00:23,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5115225b7a59d832f569746a9b524ee8, disabling compactions & flushes 2024-11-21T22:00:23,624 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:23,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:23,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. after waiting 0 ms 2024-11-21T22:00:23,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:23,624 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:23,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5115225b7a59d832f569746a9b524ee8: Waiting for close lock at 1732226423624Disabling compacts and flushes for region at 1732226423624Disabling writes for close at 1732226423624Writing region close event to WAL at 1732226423624Closed at 1732226423624 2024-11-21T22:00:23,625 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6fe395b08c7826341bd5d0e43b4c993a, NAME => 'test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741854_1030 (size=45) 2024-11-21T22:00:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741853_1029 (size=45) 2024-11-21T22:00:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741854_1030 (size=45) 2024-11-21T22:00:23,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741854_1030 (size=45) 2024-11-21T22:00:23,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741853_1029 (size=45) 2024-11-21T22:00:23,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741853_1029 (size=45) 2024-11-21T22:00:23,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741855_1031 (size=45) 2024-11-21T22:00:23,641 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,641 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d58deb362db49650ac0424f413c0d8a1, disabling compactions & flushes 2024-11-21T22:00:23,641 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. 2024-11-21T22:00:23,641 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. 2024-11-21T22:00:23,641 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. after waiting 0 ms 2024-11-21T22:00:23,641 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. 2024-11-21T22:00:23,641 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. 2024-11-21T22:00:23,641 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,641 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d58deb362db49650ac0424f413c0d8a1: Waiting for close lock at 1732226423641Disabling compacts and flushes for region at 1732226423641Disabling writes for close at 1732226423641Writing region close event to WAL at 1732226423641Closed at 1732226423641 2024-11-21T22:00:23,641 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 17a273c548a9d111239f960bce54c80e, disabling compactions & flushes 2024-11-21T22:00:23,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741855_1031 (size=45) 2024-11-21T22:00:23,642 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:23,642 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:23,642 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b3fe4283fabe9b42395c59e1bedb1cba, NAME => 'test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,642 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. after waiting 0 ms 2024-11-21T22:00:23,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741855_1031 (size=45) 2024-11-21T22:00:23,642 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:23,642 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:23,642 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 17a273c548a9d111239f960bce54c80e: Waiting for close lock at 1732226423641Disabling compacts and flushes for region at 1732226423641Disabling writes for close at 1732226423642 (+1 ms)Writing region close event to WAL at 1732226423642Closed at 1732226423642 2024-11-21T22:00:23,643 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => aff672d78bfa1216741748dbf2072a49, NAME => 'test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,643 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,643 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6fe395b08c7826341bd5d0e43b4c993a, disabling compactions & flushes 2024-11-21T22:00:23,643 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:23,643 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:23,643 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. after waiting 0 ms 2024-11-21T22:00:23,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:23,644 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:23,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6fe395b08c7826341bd5d0e43b4c993a: Waiting for close lock at 1732226423643Disabling compacts and flushes for region at 1732226423643Disabling writes for close at 1732226423644 (+1 ms)Writing region close event to WAL at 1732226423644Closed at 1732226423644 2024-11-21T22:00:23,645 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9442f3a6c1cf81c461647906e5c8134a, NAME => 'test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741857_1033 (size=45) 2024-11-21T22:00:23,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741857_1033 (size=45) 2024-11-21T22:00:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741856_1032 (size=45) 2024-11-21T22:00:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741856_1032 (size=45) 2024-11-21T22:00:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741856_1032 (size=45) 2024-11-21T22:00:23,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741857_1033 (size=45) 2024-11-21T22:00:23,662 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,662 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b3fe4283fabe9b42395c59e1bedb1cba, disabling compactions & flushes 2024-11-21T22:00:23,662 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:23,662 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:23,662 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. after waiting 0 ms 2024-11-21T22:00:23,662 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:23,662 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,662 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:23,663 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing aff672d78bfa1216741748dbf2072a49, disabling compactions & flushes 2024-11-21T22:00:23,663 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b3fe4283fabe9b42395c59e1bedb1cba: Waiting for close lock at 1732226423662Disabling compacts and flushes for region at 1732226423662Disabling writes for close at 1732226423662Writing region close event to WAL at 1732226423662Closed at 1732226423662 2024-11-21T22:00:23,663 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:23,663 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:23,663 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8f8584ecc213f380a7bf9bef219b6473, NAME => 'test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,663 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. after waiting 0 ms 2024-11-21T22:00:23,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:23,664 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:23,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for aff672d78bfa1216741748dbf2072a49: Waiting for close lock at 1732226423663Disabling compacts and flushes for region at 1732226423663Disabling writes for close at 1732226423664 (+1 ms)Writing region close event to WAL at 1732226423664Closed at 1732226423664 2024-11-21T22:00:23,665 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 50e71a63bd2106ad870772d459d21d6b, NAME => 'test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741858_1034 (size=45) 2024-11-21T22:00:23,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741858_1034 (size=45) 2024-11-21T22:00:23,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741858_1034 (size=45) 2024-11-21T22:00:23,670 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,670 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9442f3a6c1cf81c461647906e5c8134a, disabling compactions & flushes 2024-11-21T22:00:23,670 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:23,671 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:23,671 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. after waiting 0 ms 2024-11-21T22:00:23,671 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:23,671 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:23,671 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9442f3a6c1cf81c461647906e5c8134a: Waiting for close lock at 1732226423670Disabling compacts and flushes for region at 1732226423670Disabling writes for close at 1732226423671 (+1 ms)Writing region close event to WAL at 1732226423671Closed at 1732226423671 2024-11-21T22:00:23,672 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6a3d82d7cb61c550b508756346794599, NAME => 'test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741860_1036 (size=45) 2024-11-21T22:00:23,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741860_1036 (size=45) 2024-11-21T22:00:23,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741860_1036 (size=45) 2024-11-21T22:00:23,698 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,698 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8f8584ecc213f380a7bf9bef219b6473, disabling compactions & flushes 2024-11-21T22:00:23,698 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. 2024-11-21T22:00:23,698 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. 2024-11-21T22:00:23,698 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. after waiting 0 ms 2024-11-21T22:00:23,698 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. 2024-11-21T22:00:23,698 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. 2024-11-21T22:00:23,698 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8f8584ecc213f380a7bf9bef219b6473: Waiting for close lock at 1732226423698Disabling compacts and flushes for region at 1732226423698Disabling writes for close at 1732226423698Writing region close event to WAL at 1732226423698Closed at 1732226423698 2024-11-21T22:00:23,699 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d822a89e8f6f1e186226929ffc5148ff, NAME => 'test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741859_1035 (size=45) 2024-11-21T22:00:23,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741859_1035 (size=45) 2024-11-21T22:00:23,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741859_1035 (size=45) 2024-11-21T22:00:23,720 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,721 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 50e71a63bd2106ad870772d459d21d6b, disabling compactions & flushes 2024-11-21T22:00:23,721 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:23,721 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:23,721 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. after waiting 0 ms 2024-11-21T22:00:23,721 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:23,721 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:23,721 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 50e71a63bd2106ad870772d459d21d6b: Waiting for close lock at 1732226423721Disabling compacts and flushes for region at 1732226423721Disabling writes for close at 1732226423721Writing region close event to WAL at 1732226423721Closed at 1732226423721 2024-11-21T22:00:23,722 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 15ef09871c0bb53efdb85d5008b7e94c, NAME => 'test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419 2024-11-21T22:00:23,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741861_1037 (size=45) 2024-11-21T22:00:23,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741861_1037 (size=45) 2024-11-21T22:00:23,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741861_1037 (size=45) 2024-11-21T22:00:23,727 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,728 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6a3d82d7cb61c550b508756346794599, disabling compactions & flushes 2024-11-21T22:00:23,728 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:23,728 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:23,728 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. after waiting 0 ms 2024-11-21T22:00:23,728 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:23,728 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:23,728 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6a3d82d7cb61c550b508756346794599: Waiting for close lock at 1732226423727Disabling compacts and flushes for region at 1732226423727Disabling writes for close at 1732226423728 (+1 ms)Writing region close event to WAL at 1732226423728Closed at 1732226423728 2024-11-21T22:00:23,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741862_1038 (size=45) 2024-11-21T22:00:23,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741862_1038 (size=45) 2024-11-21T22:00:23,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741862_1038 (size=45) 2024-11-21T22:00:23,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741863_1039 (size=42) 2024-11-21T22:00:23,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741863_1039 (size=42) 2024-11-21T22:00:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T22:00:23,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741863_1039 (size=42) 2024-11-21T22:00:23,768 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:23,768 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 15ef09871c0bb53efdb85d5008b7e94c, disabling compactions & flushes 2024-11-21T22:00:23,768 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:23,768 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:23,768 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. after waiting 0 ms 2024-11-21T22:00:23,768 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:23,768 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:23,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 15ef09871c0bb53efdb85d5008b7e94c: Waiting for close lock at 1732226423768Disabling compacts and flushes for region at 1732226423768Disabling writes for close at 1732226423768Writing region close event to WAL at 1732226423768Closed at 1732226423768 2024-11-21T22:00:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T22:00:24,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d822a89e8f6f1e186226929ffc5148ff, disabling compactions & flushes 2024-11-21T22:00:24,152 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:24,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:24,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. after waiting 0 ms 2024-11-21T22:00:24,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:24,152 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:24,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d822a89e8f6f1e186226929ffc5148ff: Waiting for close lock at 1732226424152Disabling compacts and flushes for region at 1732226424152Disabling writes for close at 1732226424152Writing region close event to WAL at 1732226424152Closed at 1732226424152 2024-11-21T22:00:24,155 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T22:00:24,162 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,162 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,163 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,163 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,163 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,164 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,164 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,164 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,164 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,164 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,165 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,165 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,165 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,165 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,166 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,166 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,166 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,166 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,166 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,167 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,167 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,167 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,167 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,167 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,168 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,168 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732226424156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226424156"}]},"ts":"1732226424156"} 2024-11-21T22:00:24,212 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-21T22:00:24,215 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T22:00:24,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732226424215"}]},"ts":"1732226424215"} 2024-11-21T22:00:24,222 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-21T22:00:24,223 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T22:00:24,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T22:00:24,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T22:00:24,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T22:00:24,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T22:00:24,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T22:00:24,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T22:00:24,242 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T22:00:24,242 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T22:00:24,242 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T22:00:24,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T22:00:24,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f98ec44fefd547dcab3f48a26fc1df58, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5737853cedadc75f4ba670c884671cc8, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37682a4b3efbf9497991cc6bf0d15002, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=063f9af8e6526f98d7dbe9567a19314c, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cc4fa100e1a67451c5548a58b0b04f47, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46ca3887dfa777a1bf18ad3f42c5ddfb, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ad08ad34efe19c87e38729ba3c5e49f, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=679d18a37ac81fe200738b66f744f5cf, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e151130b05e8b6cf9cbcab7d52f2339d, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5115225b7a59d832f569746a9b524ee8, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17a273c548a9d111239f960bce54c80e, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6fe395b08c7826341bd5d0e43b4c993a, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aff672d78bfa1216741748dbf2072a49, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9442f3a6c1cf81c461647906e5c8134a, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=50e71a63bd2106ad870772d459d21d6b, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a3d82d7cb61c550b508756346794599, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d822a89e8f6f1e186226929ffc5148ff, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN}] 2024-11-21T22:00:24,254 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:24,256 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN; state=OFFLINE, location=2a884b411602,34991,1732226420916; forceNewPlan=false, retain=false 2024-11-21T22:00:24,257 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d822a89e8f6f1e186226929ffc5148ff, ASSIGN 2024-11-21T22:00:24,258 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d822a89e8f6f1e186226929ffc5148ff, ASSIGN; state=OFFLINE, location=2a884b411602,36681,1732226420789; forceNewPlan=false, retain=false 2024-11-21T22:00:24,260 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a3d82d7cb61c550b508756346794599, ASSIGN 2024-11-21T22:00:24,261 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6a3d82d7cb61c550b508756346794599, ASSIGN; state=OFFLINE, location=2a884b411602,45209,1732226420988; forceNewPlan=false, retain=false 2024-11-21T22:00:24,263 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=50e71a63bd2106ad870772d459d21d6b, ASSIGN 2024-11-21T22:00:24,265 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=50e71a63bd2106ad870772d459d21d6b, ASSIGN; state=OFFLINE, location=2a884b411602,36681,1732226420789; forceNewPlan=false, retain=false 2024-11-21T22:00:24,266 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:24,267 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN; state=OFFLINE, location=2a884b411602,34991,1732226420916; forceNewPlan=false, retain=false 2024-11-21T22:00:24,268 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9442f3a6c1cf81c461647906e5c8134a, ASSIGN 2024-11-21T22:00:24,270 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9442f3a6c1cf81c461647906e5c8134a, ASSIGN; state=OFFLINE, location=2a884b411602,45209,1732226420988; forceNewPlan=false, retain=false 2024-11-21T22:00:24,271 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aff672d78bfa1216741748dbf2072a49, ASSIGN 2024-11-21T22:00:24,272 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=aff672d78bfa1216741748dbf2072a49, ASSIGN; state=OFFLINE, location=2a884b411602,36681,1732226420789; forceNewPlan=false, retain=false 2024-11-21T22:00:24,274 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:24,276 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN; state=OFFLINE, location=2a884b411602,34991,1732226420916; forceNewPlan=false, retain=false 2024-11-21T22:00:24,278 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6fe395b08c7826341bd5d0e43b4c993a, ASSIGN 2024-11-21T22:00:24,280 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6fe395b08c7826341bd5d0e43b4c993a, ASSIGN; state=OFFLINE, location=2a884b411602,45209,1732226420988; forceNewPlan=false, retain=false 2024-11-21T22:00:24,282 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17a273c548a9d111239f960bce54c80e, ASSIGN 2024-11-21T22:00:24,284 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=17a273c548a9d111239f960bce54c80e, ASSIGN; state=OFFLINE, location=2a884b411602,36681,1732226420789; forceNewPlan=false, retain=false 2024-11-21T22:00:24,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:24,288 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN; state=OFFLINE, location=2a884b411602,34991,1732226420916; forceNewPlan=false, retain=false 2024-11-21T22:00:24,290 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5115225b7a59d832f569746a9b524ee8, ASSIGN 2024-11-21T22:00:24,292 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5115225b7a59d832f569746a9b524ee8, ASSIGN; state=OFFLINE, location=2a884b411602,45209,1732226420988; forceNewPlan=false, retain=false 2024-11-21T22:00:24,294 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:24,296 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN; state=OFFLINE, location=2a884b411602,34991,1732226420916; forceNewPlan=false, retain=false 2024-11-21T22:00:24,297 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e151130b05e8b6cf9cbcab7d52f2339d, ASSIGN 2024-11-21T22:00:24,299 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e151130b05e8b6cf9cbcab7d52f2339d, ASSIGN; state=OFFLINE, location=2a884b411602,36681,1732226420789; forceNewPlan=false, retain=false 2024-11-21T22:00:24,300 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=679d18a37ac81fe200738b66f744f5cf, ASSIGN 2024-11-21T22:00:24,302 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=679d18a37ac81fe200738b66f744f5cf, ASSIGN; state=OFFLINE, location=2a884b411602,45209,1732226420988; forceNewPlan=false, retain=false 2024-11-21T22:00:24,303 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ad08ad34efe19c87e38729ba3c5e49f, ASSIGN 2024-11-21T22:00:24,304 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9ad08ad34efe19c87e38729ba3c5e49f, ASSIGN; state=OFFLINE, location=2a884b411602,36681,1732226420789; forceNewPlan=false, retain=false 2024-11-21T22:00:24,306 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, ASSIGN 2024-11-21T22:00:24,307 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, ASSIGN; state=OFFLINE, location=2a884b411602,34991,1732226420916; forceNewPlan=false, retain=false 2024-11-21T22:00:24,309 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46ca3887dfa777a1bf18ad3f42c5ddfb, ASSIGN 2024-11-21T22:00:24,310 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=46ca3887dfa777a1bf18ad3f42c5ddfb, ASSIGN; state=OFFLINE, location=2a884b411602,45209,1732226420988; forceNewPlan=false, retain=false 2024-11-21T22:00:24,311 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cc4fa100e1a67451c5548a58b0b04f47, ASSIGN 2024-11-21T22:00:24,313 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cc4fa100e1a67451c5548a58b0b04f47, ASSIGN; state=OFFLINE, location=2a884b411602,36681,1732226420789; forceNewPlan=false, retain=false 2024-11-21T22:00:24,314 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:24,316 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN; state=OFFLINE, location=2a884b411602,34991,1732226420916; forceNewPlan=false, retain=false 2024-11-21T22:00:24,317 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=063f9af8e6526f98d7dbe9567a19314c, ASSIGN 2024-11-21T22:00:24,319 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=063f9af8e6526f98d7dbe9567a19314c, ASSIGN; state=OFFLINE, location=2a884b411602,45209,1732226420988; forceNewPlan=false, retain=false 2024-11-21T22:00:24,320 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37682a4b3efbf9497991cc6bf0d15002, ASSIGN 2024-11-21T22:00:24,321 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=37682a4b3efbf9497991cc6bf0d15002, ASSIGN; state=OFFLINE, location=2a884b411602,36681,1732226420789; forceNewPlan=false, retain=false 2024-11-21T22:00:24,323 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:24,325 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN; state=OFFLINE, location=2a884b411602,34991,1732226420916; forceNewPlan=false, retain=false 2024-11-21T22:00:24,326 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:24,328 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN; state=OFFLINE, location=2a884b411602,34991,1732226420916; forceNewPlan=false, retain=false 2024-11-21T22:00:24,329 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5737853cedadc75f4ba670c884671cc8, ASSIGN 2024-11-21T22:00:24,331 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5737853cedadc75f4ba670c884671cc8, ASSIGN; state=OFFLINE, location=2a884b411602,36681,1732226420789; forceNewPlan=false, retain=false 2024-11-21T22:00:24,332 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f98ec44fefd547dcab3f48a26fc1df58, ASSIGN 2024-11-21T22:00:24,334 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f98ec44fefd547dcab3f48a26fc1df58, ASSIGN; state=OFFLINE, location=2a884b411602,45209,1732226420988; forceNewPlan=false, retain=false 2024-11-21T22:00:24,409 INFO [2a884b411602:38029 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-21T22:00:24,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=6a3d82d7cb61c550b508756346794599, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=9442f3a6c1cf81c461647906e5c8134a, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=5115225b7a59d832f569746a9b524ee8, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=679d18a37ac81fe200738b66f744f5cf, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,413 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=063f9af8e6526f98d7dbe9567a19314c, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6a3d82d7cb61c550b508756346794599, ASSIGN because future has completed 2024-11-21T22:00:24,413 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,414 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d822a89e8f6f1e186226929ffc5148ff, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a3d82d7cb61c550b508756346794599, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:24,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9442f3a6c1cf81c461647906e5c8134a, ASSIGN because future has completed 2024-11-21T22:00:24,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6fe395b08c7826341bd5d0e43b4c993a, ASSIGN because future has completed 2024-11-21T22:00:24,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5115225b7a59d832f569746a9b524ee8, ASSIGN because future has completed 2024-11-21T22:00:24,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=679d18a37ac81fe200738b66f744f5cf, ASSIGN because future has completed 2024-11-21T22:00:24,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 679d18a37ac81fe200738b66f744f5cf, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:24,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=46ca3887dfa777a1bf18ad3f42c5ddfb, ASSIGN because future has completed 2024-11-21T22:00:24,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=063f9af8e6526f98d7dbe9567a19314c, ASSIGN because future has completed 2024-11-21T22:00:24,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f98ec44fefd547dcab3f48a26fc1df58, ASSIGN because future has completed 2024-11-21T22:00:24,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d822a89e8f6f1e186226929ffc5148ff, ASSIGN because future has completed 2024-11-21T22:00:24,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d822a89e8f6f1e186226929ffc5148ff, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:24,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f98ec44fefd547dcab3f48a26fc1df58, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:24,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 063f9af8e6526f98d7dbe9567a19314c, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:24,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46ca3887dfa777a1bf18ad3f42c5ddfb, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:24,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5115225b7a59d832f569746a9b524ee8, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:24,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fe395b08c7826341bd5d0e43b4c993a, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:24,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9442f3a6c1cf81c461647906e5c8134a, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:24,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=50e71a63bd2106ad870772d459d21d6b, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,465 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=aff672d78bfa1216741748dbf2072a49, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=17a273c548a9d111239f960bce54c80e, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=50e71a63bd2106ad870772d459d21d6b, ASSIGN because future has completed 2024-11-21T22:00:24,467 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50e71a63bd2106ad870772d459d21d6b, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:24,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=aff672d78bfa1216741748dbf2072a49, ASSIGN because future has completed 2024-11-21T22:00:24,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=17a273c548a9d111239f960bce54c80e, ASSIGN because future has completed 2024-11-21T22:00:24,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17a273c548a9d111239f960bce54c80e, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:24,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e151130b05e8b6cf9cbcab7d52f2339d, ASSIGN because future has completed 2024-11-21T22:00:24,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9ad08ad34efe19c87e38729ba3c5e49f, ASSIGN because future has completed 2024-11-21T22:00:24,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ad08ad34efe19c87e38729ba3c5e49f, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:24,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e151130b05e8b6cf9cbcab7d52f2339d, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:24,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure aff672d78bfa1216741748dbf2072a49, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:24,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=37682a4b3efbf9497991cc6bf0d15002, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=5737853cedadc75f4ba670c884671cc8, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,499 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cc4fa100e1a67451c5548a58b0b04f47, ASSIGN because future has completed 2024-11-21T22:00:24,499 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc4fa100e1a67451c5548a58b0b04f47, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:24,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=37682a4b3efbf9497991cc6bf0d15002, ASSIGN because future has completed 2024-11-21T22:00:24,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5737853cedadc75f4ba670c884671cc8, ASSIGN because future has completed 2024-11-21T22:00:24,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN because future has completed 2024-11-21T22:00:24,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:24,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN because future has completed 2024-11-21T22:00:24,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN because future has completed 2024-11-21T22:00:24,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:24,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:24,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5737853cedadc75f4ba670c884671cc8, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:24,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37682a4b3efbf9497991cc6bf0d15002, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:24,527 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,527 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=bb120caa77e7b0578515651fde091e69, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,528 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=95832a57de44abb16e1581ee9057cab8, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,528 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,529 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,529 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN because future has completed 2024-11-21T22:00:24,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:24,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN because future has completed 2024-11-21T22:00:24,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, ASSIGN because future has completed 2024-11-21T22:00:24,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN because future has completed 2024-11-21T22:00:24,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN because future has completed 2024-11-21T22:00:24,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:24,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN because future has completed 2024-11-21T22:00:24,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:24,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:24,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95832a57de44abb16e1581ee9057cab8, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:24,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb120caa77e7b0578515651fde091e69, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:24,569 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T22:00:24,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T22:00:24,577 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:24,577 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => f98ec44fefd547dcab3f48a26fc1df58, NAME => 'test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-21T22:00:24,578 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,578 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,578 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,578 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,581 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T22:00:24,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43495, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T22:00:24,585 INFO [StoreOpener-f98ec44fefd547dcab3f48a26fc1df58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T22:00:24,587 INFO [StoreOpener-f98ec44fefd547dcab3f48a26fc1df58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f98ec44fefd547dcab3f48a26fc1df58 columnFamilyName family 2024-11-21T22:00:24,587 DEBUG [StoreOpener-f98ec44fefd547dcab3f48a26fc1df58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,588 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:24,588 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 17a273c548a9d111239f960bce54c80e, NAME => 'test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-21T22:00:24,589 INFO [StoreOpener-f98ec44fefd547dcab3f48a26fc1df58-1 {}] regionserver.HStore(327): Store=f98ec44fefd547dcab3f48a26fc1df58/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,589 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,589 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,589 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,589 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,589 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,590 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,591 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,591 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,591 INFO [StoreOpener-17a273c548a9d111239f960bce54c80e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,591 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,593 INFO [StoreOpener-17a273c548a9d111239f960bce54c80e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17a273c548a9d111239f960bce54c80e columnFamilyName family 2024-11-21T22:00:24,593 DEBUG [StoreOpener-17a273c548a9d111239f960bce54c80e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,594 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,594 INFO [StoreOpener-17a273c548a9d111239f960bce54c80e-1 {}] regionserver.HStore(327): Store=17a273c548a9d111239f960bce54c80e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,594 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,596 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,596 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,597 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/f98ec44fefd547dcab3f48a26fc1df58/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,597 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,597 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,597 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened f98ec44fefd547dcab3f48a26fc1df58; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70304354, jitterRate=0.04761651158332825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,597 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:24,598 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for f98ec44fefd547dcab3f48a26fc1df58: Running coprocessor pre-open hook at 1732226424582Writing region info on filesystem at 1732226424582Initializing all the Stores at 1732226424584 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424584Cleaning up temporary data from old regions at 1732226424591 (+7 ms)Running coprocessor post-open hooks at 1732226424597 (+6 ms)Region opened successfully at 1732226424598 (+1 ms) 2024-11-21T22:00:24,599 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,600 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58., pid=34, masterSystemTime=1732226424569 2024-11-21T22:00:24,602 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/17a273c548a9d111239f960bce54c80e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,603 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:24,603 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:24,603 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:24,603 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 17a273c548a9d111239f960bce54c80e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62157423, jitterRate=-0.07378222048282623}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,603 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:24,603 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 679d18a37ac81fe200738b66f744f5cf, NAME => 'test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-21T22:00:24,604 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 17a273c548a9d111239f960bce54c80e: Running coprocessor pre-open hook at 1732226424590Writing region info on filesystem at 1732226424590Initializing all the Stores at 1732226424591 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424591Cleaning up temporary data from old regions at 1732226424597 (+6 ms)Running coprocessor post-open hooks at 1732226424603 (+6 ms)Region opened successfully at 1732226424604 (+1 ms) 2024-11-21T22:00:24,604 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,604 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,604 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,604 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,604 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,605 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e., pid=41, masterSystemTime=1732226424581 2024-11-21T22:00:24,607 INFO [StoreOpener-679d18a37ac81fe200738b66f744f5cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f98ec44fefd547dcab3f48a26fc1df58, server=2a884b411602,45209,1732226420988 because future has completed 2024-11-21T22:00:24,609 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:24,609 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:24,609 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:24,609 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => e151130b05e8b6cf9cbcab7d52f2339d, NAME => 'test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-21T22:00:24,610 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,610 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,610 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,610 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,610 INFO [StoreOpener-679d18a37ac81fe200738b66f744f5cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 679d18a37ac81fe200738b66f744f5cf columnFamilyName family 2024-11-21T22:00:24,610 DEBUG [StoreOpener-679d18a37ac81fe200738b66f744f5cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,611 INFO [StoreOpener-679d18a37ac81fe200738b66f744f5cf-1 {}] regionserver.HStore(327): Store=679d18a37ac81fe200738b66f744f5cf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,611 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,612 INFO [StoreOpener-e151130b05e8b6cf9cbcab7d52f2339d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,613 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,613 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=5 2024-11-21T22:00:24,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f98ec44fefd547dcab3f48a26fc1df58, server=2a884b411602,45209,1732226420988 in 176 msec 2024-11-21T22:00:24,614 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,614 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,615 INFO [StoreOpener-e151130b05e8b6cf9cbcab7d52f2339d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e151130b05e8b6cf9cbcab7d52f2339d columnFamilyName family 2024-11-21T22:00:24,616 DEBUG [StoreOpener-e151130b05e8b6cf9cbcab7d52f2339d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,616 INFO [StoreOpener-e151130b05e8b6cf9cbcab7d52f2339d-1 {}] regionserver.HStore(327): Store=e151130b05e8b6cf9cbcab7d52f2339d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,616 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f98ec44fefd547dcab3f48a26fc1df58, ASSIGN in 369 msec 2024-11-21T22:00:24,617 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,618 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,619 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,619 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=17a273c548a9d111239f960bce54c80e, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,619 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,620 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,622 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/679d18a37ac81fe200738b66f744f5cf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17a273c548a9d111239f960bce54c80e, server=2a884b411602,36681,1732226420789 because future has completed 2024-11-21T22:00:24,622 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,623 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 679d18a37ac81fe200738b66f744f5cf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63307031, jitterRate=-0.056651726365089417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,623 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:24,624 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 679d18a37ac81fe200738b66f744f5cf: Running coprocessor pre-open hook at 1732226424604Writing region info on filesystem at 1732226424604Initializing all the Stores at 1732226424606 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424606Cleaning up temporary data from old regions at 1732226424614 (+8 ms)Running coprocessor post-open hooks at 1732226424623 (+9 ms)Region opened successfully at 1732226424624 (+1 ms) 2024-11-21T22:00:24,625 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf., pid=32, masterSystemTime=1732226424569 2024-11-21T22:00:24,626 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/e151130b05e8b6cf9cbcab7d52f2339d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,627 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened e151130b05e8b6cf9cbcab7d52f2339d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58932369, jitterRate=-0.12183926999568939}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,627 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:24,627 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for e151130b05e8b6cf9cbcab7d52f2339d: Running coprocessor pre-open hook at 1732226424610Writing region info on filesystem at 1732226424610Initializing all the Stores at 1732226424612 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424612Cleaning up temporary data from old regions at 1732226424620 (+8 ms)Running coprocessor post-open hooks at 1732226424627 (+7 ms)Region opened successfully at 1732226424627 2024-11-21T22:00:24,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=21 2024-11-21T22:00:24,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 17a273c548a9d111239f960bce54c80e, server=2a884b411602,36681,1732226420789 in 148 msec 2024-11-21T22:00:24,628 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:24,628 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:24,628 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d., pid=43, masterSystemTime=1732226424581 2024-11-21T22:00:24,629 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:24,629 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 063f9af8e6526f98d7dbe9567a19314c, NAME => 'test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-21T22:00:24,629 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,629 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,629 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,629 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=17a273c548a9d111239f960bce54c80e, ASSIGN in 383 msec 2024-11-21T22:00:24,631 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:24,631 INFO [StoreOpener-063f9af8e6526f98d7dbe9567a19314c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,632 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:24,632 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:24,632 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ad08ad34efe19c87e38729ba3c5e49f, NAME => 'test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-21T22:00:24,632 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,632 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,632 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,633 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,633 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,634 INFO [StoreOpener-063f9af8e6526f98d7dbe9567a19314c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 063f9af8e6526f98d7dbe9567a19314c columnFamilyName family 2024-11-21T22:00:24,635 INFO [StoreOpener-9ad08ad34efe19c87e38729ba3c5e49f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,635 DEBUG [StoreOpener-063f9af8e6526f98d7dbe9567a19314c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,636 INFO [StoreOpener-063f9af8e6526f98d7dbe9567a19314c-1 {}] regionserver.HStore(327): Store=063f9af8e6526f98d7dbe9567a19314c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e151130b05e8b6cf9cbcab7d52f2339d, server=2a884b411602,36681,1732226420789 because future has completed 2024-11-21T22:00:24,636 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,637 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,637 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,638 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,639 INFO [StoreOpener-9ad08ad34efe19c87e38729ba3c5e49f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ad08ad34efe19c87e38729ba3c5e49f columnFamilyName family 2024-11-21T22:00:24,639 DEBUG [StoreOpener-9ad08ad34efe19c87e38729ba3c5e49f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,639 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,639 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,640 INFO [StoreOpener-9ad08ad34efe19c87e38729ba3c5e49f-1 {}] regionserver.HStore(327): Store=9ad08ad34efe19c87e38729ba3c5e49f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,641 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 679d18a37ac81fe200738b66f744f5cf, server=2a884b411602,45209,1732226420988 because future has completed 2024-11-21T22:00:24,642 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,642 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,642 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,643 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,643 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=17 2024-11-21T22:00:24,646 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure e151130b05e8b6cf9cbcab7d52f2339d, server=2a884b411602,36681,1732226420789 in 156 msec 2024-11-21T22:00:24,646 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/063f9af8e6526f98d7dbe9567a19314c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,647 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 063f9af8e6526f98d7dbe9567a19314c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74533902, jitterRate=0.11064168810844421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,647 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:24,647 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 063f9af8e6526f98d7dbe9567a19314c: Running coprocessor pre-open hook at 1732226424629Writing region info on filesystem at 1732226424630 (+1 ms)Initializing all the Stores at 1732226424631 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424631Cleaning up temporary data from old regions at 1732226424639 (+8 ms)Running coprocessor post-open hooks at 1732226424647 (+8 ms)Region opened successfully at 1732226424647 2024-11-21T22:00:24,649 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c., pid=35, masterSystemTime=1732226424569 2024-11-21T22:00:24,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e151130b05e8b6cf9cbcab7d52f2339d, ASSIGN in 401 msec 2024-11-21T22:00:24,651 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/9ad08ad34efe19c87e38729ba3c5e49f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,652 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:24,652 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:24,652 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:24,652 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 9442f3a6c1cf81c461647906e5c8134a, NAME => 'test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-21T22:00:24,652 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 9ad08ad34efe19c87e38729ba3c5e49f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75066437, jitterRate=0.11857707798480988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,652 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:24,653 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 9ad08ad34efe19c87e38729ba3c5e49f: Running coprocessor pre-open hook at 1732226424633Writing region info on filesystem at 1732226424633Initializing all the Stores at 1732226424634 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424634Cleaning up temporary data from old regions at 1732226424643 (+9 ms)Running coprocessor post-open hooks at 1732226424652 (+9 ms)Region opened successfully at 1732226424652 2024-11-21T22:00:24,653 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,653 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,653 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,653 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,654 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f., pid=42, masterSystemTime=1732226424581 2024-11-21T22:00:24,656 INFO [StoreOpener-9442f3a6c1cf81c461647906e5c8134a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=16 2024-11-21T22:00:24,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 679d18a37ac81fe200738b66f744f5cf, server=2a884b411602,45209,1732226420988 in 229 msec 2024-11-21T22:00:24,657 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:24,657 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:24,657 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:24,657 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => aff672d78bfa1216741748dbf2072a49, NAME => 'test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-21T22:00:24,658 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,658 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,658 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,658 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,658 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,659 INFO [StoreOpener-9442f3a6c1cf81c461647906e5c8134a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9442f3a6c1cf81c461647906e5c8134a columnFamilyName family 2024-11-21T22:00:24,659 DEBUG [StoreOpener-9442f3a6c1cf81c461647906e5c8134a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,659 INFO [StoreOpener-9442f3a6c1cf81c461647906e5c8134a-1 {}] regionserver.HStore(327): Store=9442f3a6c1cf81c461647906e5c8134a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,660 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,662 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ad08ad34efe19c87e38729ba3c5e49f, server=2a884b411602,36681,1732226420789 because future has completed 2024-11-21T22:00:24,663 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,663 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,664 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=679d18a37ac81fe200738b66f744f5cf, ASSIGN in 416 msec 2024-11-21T22:00:24,667 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,667 INFO [StoreOpener-aff672d78bfa1216741748dbf2072a49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,668 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. 2024-11-21T22:00:24,668 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => e8aafe78c77f56bf1d0a5753056c1486, NAME => 'test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-21T22:00:24,669 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,669 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,669 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,670 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,670 INFO [StoreOpener-aff672d78bfa1216741748dbf2072a49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aff672d78bfa1216741748dbf2072a49 columnFamilyName family 2024-11-21T22:00:24,670 DEBUG [StoreOpener-aff672d78bfa1216741748dbf2072a49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,671 INFO [StoreOpener-aff672d78bfa1216741748dbf2072a49-1 {}] regionserver.HStore(327): Store=aff672d78bfa1216741748dbf2072a49/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,671 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,673 INFO [StoreOpener-e8aafe78c77f56bf1d0a5753056c1486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,673 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,673 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/9442f3a6c1cf81c461647906e5c8134a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,673 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=15 2024-11-21T22:00:24,674 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 9442f3a6c1cf81c461647906e5c8134a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68478374, jitterRate=0.020407289266586304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 9ad08ad34efe19c87e38729ba3c5e49f, server=2a884b411602,36681,1732226420789 in 186 msec 2024-11-21T22:00:24,674 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:24,674 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 9442f3a6c1cf81c461647906e5c8134a: Running coprocessor pre-open hook at 1732226424653Writing region info on filesystem at 1732226424653Initializing all the Stores at 1732226424655 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424655Cleaning up temporary data from old regions at 1732226424664 (+9 ms)Running coprocessor post-open hooks at 1732226424674 (+10 ms)Region opened successfully at 1732226424674 2024-11-21T22:00:24,674 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,674 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,675 INFO [StoreOpener-e8aafe78c77f56bf1d0a5753056c1486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8aafe78c77f56bf1d0a5753056c1486 columnFamilyName family 2024-11-21T22:00:24,675 DEBUG [StoreOpener-e8aafe78c77f56bf1d0a5753056c1486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,675 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a., pid=39, masterSystemTime=1732226424569 2024-11-21T22:00:24,676 INFO [StoreOpener-e8aafe78c77f56bf1d0a5753056c1486-1 {}] regionserver.HStore(327): Store=e8aafe78c77f56bf1d0a5753056c1486/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,676 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,678 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,678 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,678 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:24,678 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:24,679 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:24,679 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 46ca3887dfa777a1bf18ad3f42c5ddfb, NAME => 'test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-21T22:00:24,679 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,679 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,679 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,679 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,680 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9ad08ad34efe19c87e38729ba3c5e49f, ASSIGN in 429 msec 2024-11-21T22:00:24,681 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,681 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,682 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,682 INFO [StoreOpener-46ca3887dfa777a1bf18ad3f42c5ddfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,683 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/aff672d78bfa1216741748dbf2072a49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,685 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,685 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened aff672d78bfa1216741748dbf2072a49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65671621, jitterRate=-0.021416589617729187}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,685 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:24,685 INFO [StoreOpener-46ca3887dfa777a1bf18ad3f42c5ddfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46ca3887dfa777a1bf18ad3f42c5ddfb columnFamilyName family 2024-11-21T22:00:24,685 DEBUG [StoreOpener-46ca3887dfa777a1bf18ad3f42c5ddfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,685 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for aff672d78bfa1216741748dbf2072a49: Running coprocessor pre-open hook at 1732226424658Writing region info on filesystem at 1732226424658Initializing all the Stores at 1732226424660 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424660Cleaning up temporary data from old regions at 1732226424674 (+14 ms)Running coprocessor post-open hooks at 1732226424685 (+11 ms)Region opened successfully at 1732226424685 2024-11-21T22:00:24,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9442f3a6c1cf81c461647906e5c8134a, server=2a884b411602,45209,1732226420988 because future has completed 2024-11-21T22:00:24,686 INFO [StoreOpener-46ca3887dfa777a1bf18ad3f42c5ddfb-1 {}] regionserver.HStore(327): Store=46ca3887dfa777a1bf18ad3f42c5ddfb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,686 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,688 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49., pid=44, masterSystemTime=1732226424581 2024-11-21T22:00:24,688 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,689 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,690 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,690 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,691 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:24,691 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:24,691 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/e8aafe78c77f56bf1d0a5753056c1486/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,691 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:24,692 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => cc4fa100e1a67451c5548a58b0b04f47, NAME => 'test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-21T22:00:24,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 063f9af8e6526f98d7dbe9567a19314c, server=2a884b411602,45209,1732226420988 because future has completed 2024-11-21T22:00:24,692 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,692 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,692 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened e8aafe78c77f56bf1d0a5753056c1486; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74932063, jitterRate=0.11657474935054779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,693 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:24,693 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,693 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for e8aafe78c77f56bf1d0a5753056c1486: Running coprocessor pre-open hook at 1732226424670Writing region info on filesystem at 1732226424670Initializing all the Stores at 1732226424672 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424672Cleaning up temporary data from old regions at 1732226424681 (+9 ms)Running coprocessor post-open hooks at 1732226424693 (+12 ms)Region opened successfully at 1732226424693 2024-11-21T22:00:24,693 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,694 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486., pid=53, masterSystemTime=1732226424659 2024-11-21T22:00:24,694 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,696 INFO [StoreOpener-cc4fa100e1a67451c5548a58b0b04f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,697 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. 2024-11-21T22:00:24,697 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. 2024-11-21T22:00:24,697 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:24,697 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 15ef09871c0bb53efdb85d5008b7e94c, NAME => 'test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-21T22:00:24,698 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,698 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,698 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,698 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,698 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/46ca3887dfa777a1bf18ad3f42c5ddfb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure aff672d78bfa1216741748dbf2072a49, server=2a884b411602,36681,1732226420789 because future has completed 2024-11-21T22:00:24,699 INFO [StoreOpener-cc4fa100e1a67451c5548a58b0b04f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc4fa100e1a67451c5548a58b0b04f47 columnFamilyName family 2024-11-21T22:00:24,699 DEBUG [StoreOpener-cc4fa100e1a67451c5548a58b0b04f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,700 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 46ca3887dfa777a1bf18ad3f42c5ddfb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74888003, jitterRate=0.11591820418834686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,700 INFO [StoreOpener-cc4fa100e1a67451c5548a58b0b04f47-1 {}] regionserver.HStore(327): Store=cc4fa100e1a67451c5548a58b0b04f47/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,700 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:24,700 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,700 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 46ca3887dfa777a1bf18ad3f42c5ddfb: Running coprocessor pre-open hook at 1732226424679Writing region info on filesystem at 1732226424679Initializing all the Stores at 1732226424681 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424681Cleaning up temporary data from old regions at 1732226424690 (+9 ms)Running coprocessor post-open hooks at 1732226424700 (+10 ms)Region opened successfully at 1732226424700 2024-11-21T22:00:24,700 INFO [StoreOpener-15ef09871c0bb53efdb85d5008b7e94c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,701 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb., pid=36, masterSystemTime=1732226424569 2024-11-21T22:00:24,701 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,702 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=10 2024-11-21T22:00:24,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 063f9af8e6526f98d7dbe9567a19314c, server=2a884b411602,45209,1732226420988 in 261 msec 2024-11-21T22:00:24,703 INFO [StoreOpener-15ef09871c0bb53efdb85d5008b7e94c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15ef09871c0bb53efdb85d5008b7e94c columnFamilyName family 2024-11-21T22:00:24,703 DEBUG [StoreOpener-15ef09871c0bb53efdb85d5008b7e94c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,703 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,704 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,704 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:24,704 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:24,705 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:24,705 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a3d82d7cb61c550b508756346794599, NAME => 'test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-21T22:00:24,705 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,705 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,706 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,706 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=063f9af8e6526f98d7dbe9567a19314c, ASSIGN in 459 msec 2024-11-21T22:00:24,706 INFO [StoreOpener-15ef09871c0bb53efdb85d5008b7e94c-1 {}] regionserver.HStore(327): Store=15ef09871c0bb53efdb85d5008b7e94c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,706 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,708 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,708 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,708 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,709 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,709 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46ca3887dfa777a1bf18ad3f42c5ddfb, server=2a884b411602,45209,1732226420988 because future has completed 2024-11-21T22:00:24,713 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,715 INFO [StoreOpener-6a3d82d7cb61c550b508756346794599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,717 INFO [StoreOpener-6a3d82d7cb61c550b508756346794599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3d82d7cb61c550b508756346794599 columnFamilyName family 2024-11-21T22:00:24,717 DEBUG [StoreOpener-6a3d82d7cb61c550b508756346794599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=24 2024-11-21T22:00:24,718 INFO [StoreOpener-6a3d82d7cb61c550b508756346794599-1 {}] regionserver.HStore(327): Store=6a3d82d7cb61c550b508756346794599/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure aff672d78bfa1216741748dbf2072a49, server=2a884b411602,36681,1732226420789 in 221 msec 2024-11-21T22:00:24,718 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,720 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,720 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,721 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,721 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=aff672d78bfa1216741748dbf2072a49, ASSIGN in 473 msec 2024-11-21T22:00:24,723 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/15ef09871c0bb53efdb85d5008b7e94c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,724 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/cc4fa100e1a67451c5548a58b0b04f47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,725 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,725 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 15ef09871c0bb53efdb85d5008b7e94c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62825770, jitterRate=-0.06382307410240173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,725 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:24,725 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened cc4fa100e1a67451c5548a58b0b04f47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62084867, jitterRate=-0.07486338913440704}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,725 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:24,725 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 15ef09871c0bb53efdb85d5008b7e94c: Running coprocessor pre-open hook at 1732226424698Writing region info on filesystem at 1732226424698Initializing all the Stores at 1732226424700 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424700Cleaning up temporary data from old regions at 1732226424709 (+9 ms)Running coprocessor post-open hooks at 1732226424725 (+16 ms)Region opened successfully at 1732226424725 2024-11-21T22:00:24,725 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for cc4fa100e1a67451c5548a58b0b04f47: Running coprocessor pre-open hook at 1732226424693Writing region info on filesystem at 1732226424693Initializing all the Stores at 1732226424694 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424694Cleaning up temporary data from old regions at 1732226424704 (+10 ms)Running coprocessor post-open hooks at 1732226424725 (+21 ms)Region opened successfully at 1732226424725 2024-11-21T22:00:24,726 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47., pid=45, masterSystemTime=1732226424581 2024-11-21T22:00:24,726 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c., pid=46, masterSystemTime=1732226424659 2024-11-21T22:00:24,729 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/6a3d82d7cb61c550b508756346794599/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,729 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:24,730 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:24,730 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:24,730 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => d6cc8ee4cfeffaf8625f8fdd19f2e2ff, NAME => 'test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-21T22:00:24,730 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 6a3d82d7cb61c550b508756346794599; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71226485, jitterRate=0.06135733425617218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,730 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:24,730 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,730 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 6a3d82d7cb61c550b508756346794599: Running coprocessor pre-open hook at 1732226424706Writing region info on filesystem at 1732226424706Initializing all the Stores at 1732226424708 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424708Cleaning up temporary data from old regions at 1732226424721 (+13 ms)Running coprocessor post-open hooks at 1732226424730 (+9 ms)Region opened successfully at 1732226424730 2024-11-21T22:00:24,730 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,731 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,731 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,732 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599., pid=31, masterSystemTime=1732226424569 2024-11-21T22:00:24,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=13 2024-11-21T22:00:24,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 46ca3887dfa777a1bf18ad3f42c5ddfb, server=2a884b411602,45209,1732226420988 in 281 msec 2024-11-21T22:00:24,732 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:24,732 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:24,733 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:24,733 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 50e71a63bd2106ad870772d459d21d6b, NAME => 'test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-21T22:00:24,733 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,733 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,733 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,733 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,735 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,736 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:24,736 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:24,736 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:24,736 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fe395b08c7826341bd5d0e43b4c993a, NAME => 'test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-21T22:00:24,736 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,737 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,737 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,737 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc4fa100e1a67451c5548a58b0b04f47, server=2a884b411602,36681,1732226420789 because future has completed 2024-11-21T22:00:24,739 INFO [StoreOpener-d6cc8ee4cfeffaf8625f8fdd19f2e2ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=6a3d82d7cb61c550b508756346794599, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,742 INFO [StoreOpener-d6cc8ee4cfeffaf8625f8fdd19f2e2ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cc8ee4cfeffaf8625f8fdd19f2e2ff columnFamilyName family 2024-11-21T22:00:24,742 DEBUG [StoreOpener-d6cc8ee4cfeffaf8625f8fdd19f2e2ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a3d82d7cb61c550b508756346794599, server=2a884b411602,45209,1732226420988 because future has completed 2024-11-21T22:00:24,743 INFO [StoreOpener-d6cc8ee4cfeffaf8625f8fdd19f2e2ff-1 {}] regionserver.HStore(327): Store=d6cc8ee4cfeffaf8625f8fdd19f2e2ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,743 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,744 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,745 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,745 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,746 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,747 INFO [StoreOpener-50e71a63bd2106ad870772d459d21d6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=12 2024-11-21T22:00:24,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure cc4fa100e1a67451c5548a58b0b04f47, server=2a884b411602,36681,1732226420789 in 243 msec 2024-11-21T22:00:24,748 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,749 INFO [StoreOpener-50e71a63bd2106ad870772d459d21d6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50e71a63bd2106ad870772d459d21d6b columnFamilyName family 2024-11-21T22:00:24,749 DEBUG [StoreOpener-50e71a63bd2106ad870772d459d21d6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,750 INFO [StoreOpener-50e71a63bd2106ad870772d459d21d6b-1 {}] regionserver.HStore(327): Store=50e71a63bd2106ad870772d459d21d6b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,750 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cc4fa100e1a67451c5548a58b0b04f47, ASSIGN in 502 msec 2024-11-21T22:00:24,751 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,752 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,753 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,753 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,755 INFO [StoreOpener-6fe395b08c7826341bd5d0e43b4c993a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,756 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,757 INFO [StoreOpener-6fe395b08c7826341bd5d0e43b4c993a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fe395b08c7826341bd5d0e43b4c993a columnFamilyName family 2024-11-21T22:00:24,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-21T22:00:24,757 DEBUG [StoreOpener-6fe395b08c7826341bd5d0e43b4c993a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 6a3d82d7cb61c550b508756346794599, server=2a884b411602,45209,1732226420988 in 338 msec 2024-11-21T22:00:24,759 INFO [StoreOpener-6fe395b08c7826341bd5d0e43b4c993a-1 {}] regionserver.HStore(327): Store=6fe395b08c7826341bd5d0e43b4c993a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,759 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,760 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,761 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6a3d82d7cb61c550b508756346794599, ASSIGN in 513 msec 2024-11-21T22:00:24,762 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,762 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,765 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=46ca3887dfa777a1bf18ad3f42c5ddfb, ASSIGN in 517 msec 2024-11-21T22:00:24,768 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:24,772 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,772 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/50e71a63bd2106ad870772d459d21d6b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,773 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 50e71a63bd2106ad870772d459d21d6b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67020860, jitterRate=-0.0013113617897033691}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,773 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:24,775 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d6cc8ee4cfeffaf8625f8fdd19f2e2ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,775 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 50e71a63bd2106ad870772d459d21d6b: Running coprocessor pre-open hook at 1732226424734Writing region info on filesystem at 1732226424734Initializing all the Stores at 1732226424736 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424736Cleaning up temporary data from old regions at 1732226424753 (+17 ms)Running coprocessor post-open hooks at 1732226424773 (+20 ms)Region opened successfully at 1732226424774 (+1 ms) 2024-11-21T22:00:24,776 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened d6cc8ee4cfeffaf8625f8fdd19f2e2ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59195421, jitterRate=-0.11791948974132538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:24,776 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/6fe395b08c7826341bd5d0e43b4c993a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,776 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:24,776 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for d6cc8ee4cfeffaf8625f8fdd19f2e2ff: Running coprocessor pre-open hook at 1732226424731Writing region info on filesystem at 1732226424731Initializing all the Stores at 1732226424733 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424733Cleaning up temporary data from old regions at 1732226424746 (+13 ms)Running coprocessor post-open hooks at 1732226424776 (+30 ms)Region opened successfully at 1732226424776 2024-11-21T22:00:24,778 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2a884b411602,45209,1732226420988, table=test, region=9442f3a6c1cf81c461647906e5c8134a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T22:00:24,778 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff., pid=52, masterSystemTime=1732226424659 2024-11-21T22:00:24,778 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b., pid=40, masterSystemTime=1732226424581 2024-11-21T22:00:24,778 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 6fe395b08c7826341bd5d0e43b4c993a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75385807, jitterRate=0.123336061835289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,778 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:24,778 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 6fe395b08c7826341bd5d0e43b4c993a: Running coprocessor pre-open hook at 1732226424737Writing region info on filesystem at 1732226424737Initializing all the Stores at 1732226424739 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424739Cleaning up temporary data from old regions at 1732226424762 (+23 ms)Running coprocessor post-open hooks at 1732226424778 (+16 ms)Region opened successfully at 1732226424778 2024-11-21T22:00:24,780 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a., pid=38, masterSystemTime=1732226424569 2024-11-21T22:00:24,781 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:24,781 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:24,781 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. 2024-11-21T22:00:24,782 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => bb120caa77e7b0578515651fde091e69, NAME => 'test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-21T22:00:24,782 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,782 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,783 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,783 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,786 INFO [StoreOpener-bb120caa77e7b0578515651fde091e69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,789 INFO [StoreOpener-bb120caa77e7b0578515651fde091e69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb120caa77e7b0578515651fde091e69 columnFamilyName family 2024-11-21T22:00:24,790 DEBUG [StoreOpener-bb120caa77e7b0578515651fde091e69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,790 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:24,791 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:24,791 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:24,791 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 5115225b7a59d832f569746a9b524ee8, NAME => 'test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-21T22:00:24,791 INFO [StoreOpener-bb120caa77e7b0578515651fde091e69-1 {}] regionserver.HStore(327): Store=bb120caa77e7b0578515651fde091e69/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,792 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,792 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,792 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:24,792 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,792 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:24,792 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,792 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,792 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:24,793 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 37682a4b3efbf9497991cc6bf0d15002, NAME => 'test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-21T22:00:24,793 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,793 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,794 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,794 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,794 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,794 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,794 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2a884b411602,34991,1732226420916, table=test, region=15ef09871c0bb53efdb85d5008b7e94c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T22:00:24,794 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,794 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,795 INFO [StoreOpener-5115225b7a59d832f569746a9b524ee8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,797 INFO [StoreOpener-37682a4b3efbf9497991cc6bf0d15002-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,797 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,798 INFO [StoreOpener-5115225b7a59d832f569746a9b524ee8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5115225b7a59d832f569746a9b524ee8 columnFamilyName family 2024-11-21T22:00:24,798 DEBUG [StoreOpener-5115225b7a59d832f569746a9b524ee8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,799 INFO [StoreOpener-5115225b7a59d832f569746a9b524ee8-1 {}] regionserver.HStore(327): Store=5115225b7a59d832f569746a9b524ee8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,799 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,800 INFO [StoreOpener-37682a4b3efbf9497991cc6bf0d15002-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37682a4b3efbf9497991cc6bf0d15002 columnFamilyName family 2024-11-21T22:00:24,800 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,800 DEBUG [StoreOpener-37682a4b3efbf9497991cc6bf0d15002-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,801 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,801 INFO [StoreOpener-37682a4b3efbf9497991cc6bf0d15002-1 {}] regionserver.HStore(327): Store=37682a4b3efbf9497991cc6bf0d15002/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,801 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,802 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,802 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,802 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/bb120caa77e7b0578515651fde091e69/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,802 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,803 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,803 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened bb120caa77e7b0578515651fde091e69; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66860374, jitterRate=-0.003702789545059204}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,803 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:24,803 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for bb120caa77e7b0578515651fde091e69: Running coprocessor pre-open hook at 1732226424783Writing region info on filesystem at 1732226424783Initializing all the Stores at 1732226424785 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424785Cleaning up temporary data from old regions at 1732226424794 (+9 ms)Running coprocessor post-open hooks at 1732226424803 (+9 ms)Region opened successfully at 1732226424803 2024-11-21T22:00:24,804 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,804 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,805 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,805 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69., pid=56, masterSystemTime=1732226424659 2024-11-21T22:00:24,807 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-11-21T22:00:24,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,34991,1732226420916 in 240 msec 2024-11-21T22:00:24,812 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. 2024-11-21T22:00:24,812 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. 2024-11-21T22:00:24,812 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. 2024-11-21T22:00:24,812 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f8584ecc213f380a7bf9bef219b6473, NAME => 'test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-21T22:00:24,812 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/5115225b7a59d832f569746a9b524ee8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,813 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,813 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,813 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 5115225b7a59d832f569746a9b524ee8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74581310, jitterRate=0.11134812235832214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,814 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:24,814 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,814 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 5115225b7a59d832f569746a9b524ee8: Running coprocessor pre-open hook at 1732226424793Writing region info on filesystem at 1732226424793Initializing all the Stores at 1732226424795 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424795Cleaning up temporary data from old regions at 1732226424802 (+7 ms)Running coprocessor post-open hooks at 1732226424814 (+12 ms)Region opened successfully at 1732226424814 2024-11-21T22:00:24,814 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,819 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8., pid=37, masterSystemTime=1732226424569 2024-11-21T22:00:24,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN in 565 msec 2024-11-21T22:00:24,821 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=bb120caa77e7b0578515651fde091e69, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,821 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/37682a4b3efbf9497991cc6bf0d15002/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,823 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 37682a4b3efbf9497991cc6bf0d15002; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67108557, jitterRate=-4.5746564865112305E-6}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,823 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:24,823 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 37682a4b3efbf9497991cc6bf0d15002: Running coprocessor pre-open hook at 1732226424795Writing region info on filesystem at 1732226424795Initializing all the Stores at 1732226424797 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424797Cleaning up temporary data from old regions at 1732226424804 (+7 ms)Running coprocessor post-open hooks at 1732226424823 (+19 ms)Region opened successfully at 1732226424823 2024-11-21T22:00:24,823 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:24,823 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:24,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb120caa77e7b0578515651fde091e69, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:24,826 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002., pid=50, masterSystemTime=1732226424581 2024-11-21T22:00:24,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5115225b7a59d832f569746a9b524ee8, server=2a884b411602,45209,1732226420988 because future has completed 2024-11-21T22:00:24,832 INFO [StoreOpener-8f8584ecc213f380a7bf9bef219b6473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,834 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:24,834 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:24,834 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:24,835 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => d822a89e8f6f1e186226929ffc5148ff, NAME => 'test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-21T22:00:24,835 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,836 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,836 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,836 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,839 INFO [StoreOpener-8f8584ecc213f380a7bf9bef219b6473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f8584ecc213f380a7bf9bef219b6473 columnFamilyName family 2024-11-21T22:00:24,839 DEBUG [StoreOpener-8f8584ecc213f380a7bf9bef219b6473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,840 INFO [StoreOpener-8f8584ecc213f380a7bf9bef219b6473-1 {}] regionserver.HStore(327): Store=8f8584ecc213f380a7bf9bef219b6473/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=19 2024-11-21T22:00:24,840 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 5115225b7a59d832f569746a9b524ee8, server=2a884b411602,45209,1732226420988 in 388 msec 2024-11-21T22:00:24,841 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,842 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,843 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,843 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5115225b7a59d832f569746a9b524ee8, ASSIGN in 596 msec 2024-11-21T22:00:24,846 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,846 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37682a4b3efbf9497991cc6bf0d15002, server=2a884b411602,36681,1732226420789 because future has completed 2024-11-21T22:00:24,852 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/8f8584ecc213f380a7bf9bef219b6473/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,853 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 8f8584ecc213f380a7bf9bef219b6473; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65659769, jitterRate=-0.02159319818019867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,853 INFO [StoreOpener-d822a89e8f6f1e186226929ffc5148ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,853 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:24,854 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 8f8584ecc213f380a7bf9bef219b6473: Running coprocessor pre-open hook at 1732226424815Writing region info on filesystem at 1732226424815Initializing all the Stores at 1732226424816 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424816Cleaning up temporary data from old regions at 1732226424843 (+27 ms)Running coprocessor post-open hooks at 1732226424853 (+10 ms)Region opened successfully at 1732226424853 2024-11-21T22:00:24,855 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473., pid=48, masterSystemTime=1732226424659 2024-11-21T22:00:24,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=18 2024-11-21T22:00:24,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure bb120caa77e7b0578515651fde091e69, server=2a884b411602,34991,1732226420916 in 299 msec 2024-11-21T22:00:24,857 INFO [StoreOpener-d822a89e8f6f1e186226929ffc5148ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d822a89e8f6f1e186226929ffc5148ff columnFamilyName family 2024-11-21T22:00:24,858 DEBUG [StoreOpener-d822a89e8f6f1e186226929ffc5148ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,858 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. 2024-11-21T22:00:24,858 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. 2024-11-21T22:00:24,858 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:24,859 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => b3fe4283fabe9b42395c59e1bedb1cba, NAME => 'test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-21T22:00:24,859 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,859 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,859 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,859 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN in 611 msec 2024-11-21T22:00:24,860 INFO [StoreOpener-d822a89e8f6f1e186226929ffc5148ff-1 {}] regionserver.HStore(327): Store=d822a89e8f6f1e186226929ffc5148ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,860 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,861 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=8f8584ecc213f380a7bf9bef219b6473, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,862 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,863 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,863 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,863 INFO [StoreOpener-b3fe4283fabe9b42395c59e1bedb1cba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:24,867 INFO [StoreOpener-b3fe4283fabe9b42395c59e1bedb1cba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3fe4283fabe9b42395c59e1bedb1cba columnFamilyName family 2024-11-21T22:00:24,867 DEBUG [StoreOpener-b3fe4283fabe9b42395c59e1bedb1cba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,867 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,868 INFO [StoreOpener-b3fe4283fabe9b42395c59e1bedb1cba-1 {}] regionserver.HStore(327): Store=b3fe4283fabe9b42395c59e1bedb1cba/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,868 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,870 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,871 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,872 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,872 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d822a89e8f6f1e186226929ffc5148ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,872 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=9 2024-11-21T22:00:24,873 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened d822a89e8f6f1e186226929ffc5148ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59057326, jitterRate=-0.11997726559638977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 37682a4b3efbf9497991cc6bf0d15002, server=2a884b411602,36681,1732226420789 in 344 msec 2024-11-21T22:00:24,873 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:24,873 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for d822a89e8f6f1e186226929ffc5148ff: Running coprocessor pre-open hook at 1732226424836Writing region info on filesystem at 1732226424836Initializing all the Stores at 1732226424838 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424841 (+3 ms)Cleaning up temporary data from old regions at 1732226424863 (+22 ms)Running coprocessor post-open hooks at 1732226424873 (+10 ms)Region opened successfully at 1732226424873 2024-11-21T22:00:24,875 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff., pid=33, masterSystemTime=1732226424581 2024-11-21T22:00:24,877 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=37682a4b3efbf9497991cc6bf0d15002, ASSIGN in 628 msec 2024-11-21T22:00:24,886 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:24,886 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:24,886 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:24,886 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 5737853cedadc75f4ba670c884671cc8, NAME => 'test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-21T22:00:24,886 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2a884b411602,45209,1732226420988, table=test, region=9442f3a6c1cf81c461647906e5c8134a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T22:00:24,887 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,887 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,887 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,887 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,890 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/b3fe4283fabe9b42395c59e1bedb1cba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,891 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened b3fe4283fabe9b42395c59e1bedb1cba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72645777, jitterRate=0.08250643312931061}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,891 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:24,892 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for b3fe4283fabe9b42395c59e1bedb1cba: Running coprocessor pre-open hook at 1732226424860Writing region info on filesystem at 1732226424860Initializing all the Stores at 1732226424861 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424861Cleaning up temporary data from old regions at 1732226424872 (+11 ms)Running coprocessor post-open hooks at 1732226424891 (+19 ms)Region opened successfully at 1732226424891 2024-11-21T22:00:24,893 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba., pid=47, masterSystemTime=1732226424659 2024-11-21T22:00:24,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=26 2024-11-21T22:00:24,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,34991,1732226420916 in 370 msec 2024-11-21T22:00:24,900 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:24,900 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:24,900 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:24,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN in 648 msec 2024-11-21T22:00:24,901 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 30e7bcb84c1a622f86b573b1a579eabb, NAME => 'test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-21T22:00:24,901 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,901 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,901 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,901 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:24,908 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,911 INFO [StoreOpener-5737853cedadc75f4ba670c884671cc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d822a89e8f6f1e186226929ffc5148ff, server=2a884b411602,36681,1732226420789 because future has completed 2024-11-21T22:00:24,914 INFO [StoreOpener-5737853cedadc75f4ba670c884671cc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5737853cedadc75f4ba670c884671cc8 columnFamilyName family 2024-11-21T22:00:24,914 DEBUG [StoreOpener-5737853cedadc75f4ba670c884671cc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,915 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2a884b411602,34991,1732226420916, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T22:00:24,915 INFO [StoreOpener-5737853cedadc75f4ba670c884671cc8-1 {}] regionserver.HStore(327): Store=5737853cedadc75f4ba670c884671cc8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,915 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2a884b411602,34991,1732226420916, table=test, region=15ef09871c0bb53efdb85d5008b7e94c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T22:00:24,916 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,917 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,918 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,919 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,919 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,922 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,923 INFO [StoreOpener-30e7bcb84c1a622f86b573b1a579eabb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,927 INFO [StoreOpener-30e7bcb84c1a622f86b573b1a579eabb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30e7bcb84c1a622f86b573b1a579eabb columnFamilyName family 2024-11-21T22:00:24,927 DEBUG [StoreOpener-30e7bcb84c1a622f86b573b1a579eabb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,929 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/5737853cedadc75f4ba670c884671cc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,929 INFO [StoreOpener-30e7bcb84c1a622f86b573b1a579eabb-1 {}] regionserver.HStore(327): Store=30e7bcb84c1a622f86b573b1a579eabb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=29 2024-11-21T22:00:24,929 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure d822a89e8f6f1e186226929ffc5148ff, server=2a884b411602,36681,1732226420789 in 490 msec 2024-11-21T22:00:24,930 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,931 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 5737853cedadc75f4ba670c884671cc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59437319, jitterRate=-0.11431492865085602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,931 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:24,931 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 5737853cedadc75f4ba670c884671cc8: Running coprocessor pre-open hook at 1732226424887Writing region info on filesystem at 1732226424887Initializing all the Stores at 1732226424889 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424889Cleaning up temporary data from old regions at 1732226424919 (+30 ms)Running coprocessor post-open hooks at 1732226424931 (+12 ms)Region opened successfully at 1732226424931 2024-11-21T22:00:24,931 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,932 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,932 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,932 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8., pid=49, masterSystemTime=1732226424581 2024-11-21T22:00:24,934 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d822a89e8f6f1e186226929ffc5148ff, ASSIGN in 684 msec 2024-11-21T22:00:24,936 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:24,936 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:24,939 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/30e7bcb84c1a622f86b573b1a579eabb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,939 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 30e7bcb84c1a622f86b573b1a579eabb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70814871, jitterRate=0.05522380769252777}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,939 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:24,940 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 30e7bcb84c1a622f86b573b1a579eabb: Running coprocessor pre-open hook at 1732226424902Writing region info on filesystem at 1732226424902Initializing all the Stores at 1732226424904 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424904Cleaning up temporary data from old regions at 1732226424932 (+28 ms)Running coprocessor post-open hooks at 1732226424939 (+7 ms)Region opened successfully at 1732226424939 2024-11-21T22:00:24,941 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb., pid=54, masterSystemTime=1732226424659 2024-11-21T22:00:24,944 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:24,944 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:24,944 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. 2024-11-21T22:00:24,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=23 2024-11-21T22:00:24,944 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => d58deb362db49650ac0424f413c0d8a1, NAME => 'test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-21T22:00:24,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,34991,1732226420916 in 425 msec 2024-11-21T22:00:24,945 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,945 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,945 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,945 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN in 700 msec 2024-11-21T22:00:24,951 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:24,955 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:24,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5737853cedadc75f4ba670c884671cc8, server=2a884b411602,36681,1732226420789 because future has completed 2024-11-21T22:00:24,963 INFO [StoreOpener-d58deb362db49650ac0424f413c0d8a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=11 2024-11-21T22:00:24,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,34991,1732226420916 in 417 msec 2024-11-21T22:00:24,966 INFO [StoreOpener-d58deb362db49650ac0424f413c0d8a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d58deb362db49650ac0424f413c0d8a1 columnFamilyName family 2024-11-21T22:00:24,967 DEBUG [StoreOpener-d58deb362db49650ac0424f413c0d8a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,968 INFO [StoreOpener-d58deb362db49650ac0424f413c0d8a1-1 {}] regionserver.HStore(327): Store=d58deb362db49650ac0424f413c0d8a1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:24,969 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN in 720 msec 2024-11-21T22:00:24,970 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,971 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,973 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,973 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,977 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=6 2024-11-21T22:00:24,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 5737853cedadc75f4ba670c884671cc8, server=2a884b411602,36681,1732226420789 in 453 msec 2024-11-21T22:00:24,984 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d58deb362db49650ac0424f413c0d8a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:24,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5737853cedadc75f4ba670c884671cc8, ASSIGN in 736 msec 2024-11-21T22:00:24,985 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened d58deb362db49650ac0424f413c0d8a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64576791, jitterRate=-0.037730827927589417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:24,986 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:24,986 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for d58deb362db49650ac0424f413c0d8a1: Running coprocessor pre-open hook at 1732226424945Writing region info on filesystem at 1732226424945Initializing all the Stores at 1732226424947 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424947Cleaning up temporary data from old regions at 1732226424973 (+26 ms)Running coprocessor post-open hooks at 1732226424986 (+13 ms)Region opened successfully at 1732226424986 2024-11-21T22:00:24,987 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1., pid=51, masterSystemTime=1732226424659 2024-11-21T22:00:24,990 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:24,991 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. 2024-11-21T22:00:24,991 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. 2024-11-21T22:00:24,991 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:24,991 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2a884b411602,45209,1732226420988, table=test, region=6fe395b08c7826341bd5d0e43b4c993a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T22:00:24,991 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2a884b411602,45209,1732226420988, table=test, region=9442f3a6c1cf81c461647906e5c8134a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T22:00:24,992 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 95832a57de44abb16e1581ee9057cab8, NAME => 'test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-21T22:00:24,993 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:24,993 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:24,993 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:24,993 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:24,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fe395b08c7826341bd5d0e43b4c993a, server=2a884b411602,45209,1732226420988 because future has completed 2024-11-21T22:00:24,995 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d58deb362db49650ac0424f413c0d8a1, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:24,995 INFO [StoreOpener-95832a57de44abb16e1581ee9057cab8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:24,999 INFO [StoreOpener-95832a57de44abb16e1581ee9057cab8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95832a57de44abb16e1581ee9057cab8 columnFamilyName family 2024-11-21T22:00:24,999 DEBUG [StoreOpener-95832a57de44abb16e1581ee9057cab8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:24,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:25,000 INFO [StoreOpener-95832a57de44abb16e1581ee9057cab8-1 {}] regionserver.HStore(327): Store=95832a57de44abb16e1581ee9057cab8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:25,000 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:25,001 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:25,002 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:25,004 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:25,004 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:25,007 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:25,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=20 2024-11-21T22:00:25,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,34991,1732226420916 in 475 msec 2024-11-21T22:00:25,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN in 765 msec 2024-11-21T22:00:25,017 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/95832a57de44abb16e1581ee9057cab8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T22:00:25,019 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 95832a57de44abb16e1581ee9057cab8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67837235, jitterRate=0.010853573679924011}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:25,019 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:25,019 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 95832a57de44abb16e1581ee9057cab8: Running coprocessor pre-open hook at 1732226424993Writing region info on filesystem at 1732226424993Initializing all the Stores at 1732226424995 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226424995Cleaning up temporary data from old regions at 1732226425004 (+9 ms)Running coprocessor post-open hooks at 1732226425019 (+15 ms)Region opened successfully at 1732226425019 2024-11-21T22:00:25,020 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2a884b411602,34991,1732226420916, table=test, region=15ef09871c0bb53efdb85d5008b7e94c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T22:00:25,021 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8., pid=55, masterSystemTime=1732226424659 2024-11-21T22:00:25,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=22 2024-11-21T22:00:25,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fe395b08c7826341bd5d0e43b4c993a, server=2a884b411602,45209,1732226420988 in 565 msec 2024-11-21T22:00:25,024 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:25,024 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:25,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6fe395b08c7826341bd5d0e43b4c993a, ASSIGN in 779 msec 2024-11-21T22:00:25,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:25,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95832a57de44abb16e1581ee9057cab8, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:25,036 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:25,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50e71a63bd2106ad870772d459d21d6b, server=2a884b411602,36681,1732226420789 because future has completed 2024-11-21T22:00:25,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=27 2024-11-21T22:00:25,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 50e71a63bd2106ad870772d459d21d6b, server=2a884b411602,36681,1732226420789 in 576 msec 2024-11-21T22:00:25,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=50e71a63bd2106ad870772d459d21d6b, ASSIGN in 803 msec 2024-11-21T22:00:25,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=14 2024-11-21T22:00:25,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 95832a57de44abb16e1581ee9057cab8, server=2a884b411602,34991,1732226420916 in 505 msec 2024-11-21T22:00:25,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, ASSIGN in 813 msec 2024-11-21T22:00:25,064 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:25,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:25,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=30 2024-11-21T22:00:25,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,34991,1732226420916 in 562 msec 2024-11-21T22:00:25,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN in 826 msec 2024-11-21T22:00:25,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=8 2024-11-21T22:00:25,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,34991,1732226420916 in 539 msec 2024-11-21T22:00:25,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN in 837 msec 2024-11-21T22:00:25,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=25 2024-11-21T22:00:25,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 9442f3a6c1cf81c461647906e5c8134a, server=2a884b411602,45209,1732226420988 in 630 msec 2024-11-21T22:00:25,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=4 2024-11-21T22:00:25,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9442f3a6c1cf81c461647906e5c8134a, ASSIGN in 846 msec 2024-11-21T22:00:25,097 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T22:00:25,098 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732226425097"}]},"ts":"1732226425097"} 2024-11-21T22:00:25,101 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-21T22:00:25,102 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T22:00:25,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.6720 sec 2024-11-21T22:00:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T22:00:25,600 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-21T22:00:25,600 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-21T22:00:25,603 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T22:00:25,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-21T22:00:25,632 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T22:00:25,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-21T22:00:25,634 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T22:00:25,641 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-21T22:00:25,668 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-21T22:00:25,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T22:00:25,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T22:00:25,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T22:00:25,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T22:00:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e8aafe78c77f56bf1d0a5753056c1486 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d6cc8ee4cfeffaf8625f8fdd19f2e2ff changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 30e7bcb84c1a622f86b573b1a579eabb changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 95832a57de44abb16e1581ee9057cab8 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bb120caa77e7b0578515651fde091e69 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d58deb362db49650ac0424f413c0d8a1 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b3fe4283fabe9b42395c59e1bedb1cba changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8f8584ecc213f380a7bf9bef219b6473 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 15ef09871c0bb53efdb85d5008b7e94c changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5737853cedadc75f4ba670c884671cc8 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 37682a4b3efbf9497991cc6bf0d15002 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cc4fa100e1a67451c5548a58b0b04f47 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9ad08ad34efe19c87e38729ba3c5e49f changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e151130b05e8b6cf9cbcab7d52f2339d changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 17a273c548a9d111239f960bce54c80e changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region aff672d78bfa1216741748dbf2072a49 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 50e71a63bd2106ad870772d459d21d6b changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d822a89e8f6f1e186226929ffc5148ff changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f98ec44fefd547dcab3f48a26fc1df58 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 063f9af8e6526f98d7dbe9567a19314c changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 46ca3887dfa777a1bf18ad3f42c5ddfb changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 679d18a37ac81fe200738b66f744f5cf changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5115225b7a59d832f569746a9b524ee8 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6fe395b08c7826341bd5d0e43b4c993a changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9442f3a6c1cf81c461647906e5c8134a changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6a3d82d7cb61c550b508756346794599 changed from -1.0 to 0.0, refreshing cache 2024-11-21T22:00:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-21T22:00:25,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-21T22:00:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-21T22:00:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T22:00:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-11-21T22:00:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-21T22:00:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(303): server 2 has 8 regions 2024-11-21T22:00:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T22:00:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T22:00:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T22:00:25,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T22:00:25,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T22:00:25,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T22:00:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-21T22:00:25,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-21T22:00:25,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-21T22:00:26,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 642 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-21T22:00:26,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-21T22:00:26,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.HMaster(2172): balance hri=95832a57de44abb16e1581ee9057cab8, source=2a884b411602,34991,1732226420916, destination=2a884b411602,45209,1732226420988 2024-11-21T22:00:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:26,415 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:26,417 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=95832a57de44abb16e1581ee9057cab8, regionState=CLOSING, regionLocation=2a884b411602,34991,1732226420916 2024-11-21T22:00:26,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE because future has completed 2024-11-21T22:00:26,420 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T22:00:26,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 95832a57de44abb16e1581ee9057cab8, server=2a884b411602,34991,1732226420916}] 2024-11-21T22:00:26,579 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,579 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T22:00:26,579 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 95832a57de44abb16e1581ee9057cab8, disabling compactions & flushes 2024-11-21T22:00:26,579 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:26,580 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:26,580 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. after waiting 0 ms 2024-11-21T22:00:26,580 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:26,592 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/95832a57de44abb16e1581ee9057cab8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:26,596 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:26,596 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 95832a57de44abb16e1581ee9057cab8: Waiting for close lock at 1732226426579Running coprocessor pre-close hooks at 1732226426579Disabling compacts and flushes for region at 1732226426579Disabling writes for close at 1732226426580 (+1 ms)Writing region close event to WAL at 1732226426581 (+1 ms)Running coprocessor post-close hooks at 1732226426593 (+12 ms)Closed at 1732226426596 (+3 ms) 2024-11-21T22:00:26,596 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 95832a57de44abb16e1581ee9057cab8 move to 2a884b411602,45209,1732226420988 record at close sequenceid=2 2024-11-21T22:00:26,601 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,602 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=95832a57de44abb16e1581ee9057cab8, regionState=CLOSED 2024-11-21T22:00:26,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 95832a57de44abb16e1581ee9057cab8, server=2a884b411602,34991,1732226420916 because future has completed 2024-11-21T22:00:26,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-21T22:00:26,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 95832a57de44abb16e1581ee9057cab8, server=2a884b411602,34991,1732226420916 in 187 msec 2024-11-21T22:00:26,613 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE; state=CLOSED, location=2a884b411602,45209,1732226420988; forceNewPlan=false, retain=false 2024-11-21T22:00:26,764 INFO [2a884b411602:38029 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T22:00:26,764 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=95832a57de44abb16e1581ee9057cab8, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:26,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE because future has completed 2024-11-21T22:00:26,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95832a57de44abb16e1581ee9057cab8, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:26,929 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:26,929 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 95832a57de44abb16e1581ee9057cab8, NAME => 'test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-21T22:00:26,930 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,930 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:26,930 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,930 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,933 INFO [StoreOpener-95832a57de44abb16e1581ee9057cab8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,937 INFO [StoreOpener-95832a57de44abb16e1581ee9057cab8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95832a57de44abb16e1581ee9057cab8 columnFamilyName family 2024-11-21T22:00:26,937 DEBUG [StoreOpener-95832a57de44abb16e1581ee9057cab8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:26,939 INFO [StoreOpener-95832a57de44abb16e1581ee9057cab8-1 {}] regionserver.HStore(327): Store=95832a57de44abb16e1581ee9057cab8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:26,939 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,940 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,943 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,944 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,944 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,947 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,949 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 95832a57de44abb16e1581ee9057cab8; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64039959, jitterRate=-0.04573024809360504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:26,950 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:26,950 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 95832a57de44abb16e1581ee9057cab8: Running coprocessor pre-open hook at 1732226426930Writing region info on filesystem at 1732226426930Initializing all the Stores at 1732226426932 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226426932Cleaning up temporary data from old regions at 1732226426944 (+12 ms)Running coprocessor post-open hooks at 1732226426950 (+6 ms)Region opened successfully at 1732226426950 2024-11-21T22:00:26,951 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8., pid=59, masterSystemTime=1732226426922 2024-11-21T22:00:26,956 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:26,956 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:26,957 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, openSeqNum=5, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:26,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95832a57de44abb16e1581ee9057cab8, server=2a884b411602,45209,1732226420988 because future has completed 2024-11-21T22:00:26,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-21T22:00:26,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 95832a57de44abb16e1581ee9057cab8, server=2a884b411602,45209,1732226420988 in 197 msec 2024-11-21T22:00:26,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE in 560 msec 2024-11-21T22:00:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-21T22:00:27,018 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T22:00:27,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-21T22:00:27,048 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=383, OpenFileDescriptor=647, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=4369 2024-11-21T22:00:27,051 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-21T22:00:27,052 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-21T22:00:27,053 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 2a884b411602,34991,1732226420916 2024-11-21T22:00:27,054 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 2a884b411602,34991,1732226420916: Simulated kill ***** 2024-11-21T22:00:27,054 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T22:00:27,056 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T22:00:27,059 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T22:00:27,059 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T22:00:27,060 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T22:00:27,060 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 313391104 }, "NonHeapMemoryUsage": { "committed": 153878528, "init": 7667712, "max": -1, "used": 151655912 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2a884b411602" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2a884b411602" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "2a884b411602" } ] } 2024-11-21T22:00:27,063 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38029 {}] master.MasterRpcServices(700): 2a884b411602,34991,1732226420916 reported a fatal error: ***** ABORTING region server 2a884b411602,34991,1732226420916: Simulated kill ***** 2024-11-21T22:00:27,066 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2a884b411602,34991,1732226420916' ***** 2024-11-21T22:00:27,066 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-21T22:00:27,066 INFO [RS:1;2a884b411602:34991 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T22:00:27,067 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T22:00:27,067 INFO [RS:1;2a884b411602:34991 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-21T22:00:27,067 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T22:00:27,067 INFO [RS:1;2a884b411602:34991 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-21T22:00:27,067 INFO [RS:1;2a884b411602:34991 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T22:00:27,067 INFO [RS:1;2a884b411602:34991 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2a884b411602:34991. 2024-11-21T22:00:27,068 DEBUG [RS:1;2a884b411602:34991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T22:00:27,073 DEBUG [RS:1;2a884b411602:34991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:27,080 INFO [regionserver/2a884b411602:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T22:00:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741836_1012 (size=33749) 2024-11-21T22:00:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741836_1012 (size=33749) 2024-11-21T22:00:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741836_1012 (size=33749) 2024-11-21T22:00:27,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741833_1009 (size=2774) 2024-11-21T22:00:27,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741833_1009 (size=2774) 2024-11-21T22:00:27,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741833_1009 (size=2774) 2024-11-21T22:00:27,089 DEBUG [RS:1;2a884b411602:34991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:27,090 INFO [RS:1;2a884b411602:34991 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T22:00:27,090 INFO [RS:1;2a884b411602:34991 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34991 2024-11-21T22:00:27,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T22:00:27,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2a884b411602,34991,1732226420916 2024-11-21T22:00:27,146 INFO [RS:1;2a884b411602:34991 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T22:00:27,158 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2a884b411602,34991,1732226420916] 2024-11-21T22:00:27,165 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2a884b411602,34991,1732226420916 already deleted, retry=false 2024-11-21T22:00:27,166 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2a884b411602,34991,1732226420916 on 2a884b411602,38029,1732226420059 2024-11-21T22:00:27,170 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:27,175 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 2a884b411602,34991,1732226420916 (carryingMeta=true) 2a884b411602,34991,1732226420916/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1bec0139[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T22:00:27,176 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:27,179 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true, isMeta: true 2024-11-21T22:00:27,181 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting 2024-11-21T22:00:27,183 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2a884b411602,34991,1732226420916 WAL count=1, meta=true 2024-11-21T22:00:27,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta}] 2024-11-21T22:00:27,194 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:27,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:27,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T22:00:27,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34991-0x10160b31ce40002, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T22:00:27,258 INFO [RS:1;2a884b411602:34991 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T22:00:27,259 INFO [RS:1;2a884b411602:34991 {}] regionserver.HRegionServer(1031): Exiting; stopping=2a884b411602,34991,1732226420916; zookeeper connection closed. 2024-11-21T22:00:27,259 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3efbc979 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3efbc979 2024-11-21T22:00:27,269 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T22:00:27,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36681 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-21T22:00:27,370 INFO [regionserver/2a884b411602:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T22:00:27,370 INFO [regionserver/2a884b411602:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T22:00:27,393 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, size=33.0 K (33749bytes) 2024-11-21T22:00:27,394 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-21T22:00:27,395 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-21T22:00:27,395 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta 2024-11-21T22:00:27,401 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta after 4ms 2024-11-21T22:00:27,411 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta took 19ms 2024-11-21T22:00:27,425 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-21T22:00:27,437 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta.temp 2024-11-21T22:00:27,439 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta so closing down 2024-11-21T22:00:27,439 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T22:00:27,441 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta.temp 2024-11-21T22:00:27,446 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T22:00:27,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741864_1040 (size=33508) 2024-11-21T22:00:27,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741864_1040 (size=33508) 2024-11-21T22:00:27,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741864_1040 (size=33508) 2024-11-21T22:00:27,472 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) 2024-11-21T22:00:27,479 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta.temp to hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T22:00:27,484 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 65 ms; skipped=1; WAL=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-21T22:00:27,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, journal: Splitting hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, size=33.0 K (33749bytes) at 1732226427393Finishing writing output for hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta so closing down at 1732226427439 (+46 ms)Creating recovered edits writer path=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta.temp at 1732226427441 (+2 ms)3 split writer threads finished at 1732226427446 (+5 ms)Closed recovered edits writer path=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) at 1732226427472 (+26 ms)Rename recovered edits hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta.temp to hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732226427479 (+7 ms)Processed 62 edits across 1 Regions in 65 ms; skipped=1; WAL=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732226427484 (+5 ms) 2024-11-21T22:00:27,485 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta 2024-11-21T22:00:27,486 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-21T22:00:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-21T22:00:27,500 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta to hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/oldWALs 2024-11-21T22:00:27,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-21T22:00:27,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 in 303 msec 2024-11-21T22:00:27,508 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:27,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T22:00:27,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 in 323 msec 2024-11-21T22:00:27,516 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting dir is empty, no logs to split. 2024-11-21T22:00:27,516 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2a884b411602,34991,1732226420916 WAL count=0, meta=true 2024-11-21T22:00:27,516 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2a884b411602,34991,1732226420916 WAL splitting is done? wals=0, meta=true 2024-11-21T22:00:27,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T22:00:27,522 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:27,523 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T22:00:27,674 DEBUG [2a884b411602:38029 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T22:00:27,674 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T22:00:27,674 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T22:00:27,674 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T22:00:27,674 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T22:00:27,674 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T22:00:27,674 INFO [2a884b411602:38029 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T22:00:27,674 INFO [2a884b411602:38029 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T22:00:27,674 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T22:00:27,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:27,677 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPENING 2024-11-21T22:00:27,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:27,712 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:27,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:27,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:27,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:27,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:27,713 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:27,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:27,872 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T22:00:27,873 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T22:00:27,873 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T22:00:27,875 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2a884b411602%2C36681%2C1732226420789.meta, suffix=.meta, logDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,36681,1732226420789, archiveDir=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/oldWALs, maxLogs=32 2024-11-21T22:00:27,891 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,36681,1732226420789/2a884b411602%2C36681%2C1732226420789.meta.1732226427877.meta, exclude list is [], retry=0 2024-11-21T22:00:27,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34171,DS-e91666bc-b006-4fed-88f2-dd54fc70cc22,DISK] 2024-11-21T22:00:27,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37045,DS-c564ed92-b78e-46e6-b4be-1100870e1511,DISK] 2024-11-21T22:00:27,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43557,DS-d8a9d7a5-362c-4f09-b1ac-e85456762592,DISK] 2024-11-21T22:00:27,899 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,36681,1732226420789/2a884b411602%2C36681%2C1732226420789.meta.1732226427877.meta 2024-11-21T22:00:27,900 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43385:43385),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:46549:46549)] 2024-11-21T22:00:27,900 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T22:00:27,900 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T22:00:27,900 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T22:00:27,900 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T22:00:27,900 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T22:00:27,900 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:27,901 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T22:00:27,901 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T22:00:27,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T22:00:27,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T22:00:27,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:27,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:27,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T22:00:27,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T22:00:27,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:27,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:27,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T22:00:27,912 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T22:00:27,912 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:27,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:27,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T22:00:27,914 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T22:00:27,914 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:27,914 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T22:00:27,915 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T22:00:27,916 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740 2024-11-21T22:00:27,919 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740 2024-11-21T22:00:27,920 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T22:00:27,940 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T22:00:27,945 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-21T22:00:28,032 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/info/2a765146a1dc4139af777ce21c79fa9b is 138, key is test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8./info:regioninfo/1732226424955/Put/seqid=0 2024-11-21T22:00:28,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741866_1042 (size=36381) 2024-11-21T22:00:28,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741866_1042 (size=36381) 2024-11-21T22:00:28,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741866_1042 (size=36381) 2024-11-21T22:00:28,047 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/info/2a765146a1dc4139af777ce21c79fa9b 2024-11-21T22:00:28,116 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/ns/b4b8572cca8c42e6b789f61aba493ab3 is 43, key is default/ns:d/1732226423158/Put/seqid=0 2024-11-21T22:00:28,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741867_1043 (size=5153) 2024-11-21T22:00:28,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741867_1043 (size=5153) 2024-11-21T22:00:28,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741867_1043 (size=5153) 2024-11-21T22:00:28,130 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/ns/b4b8572cca8c42e6b789f61aba493ab3 2024-11-21T22:00:28,162 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/table/bff417f50e98435da30513b0498379c0 is 40, key is test/table:state/1732226425097/Put/seqid=0 2024-11-21T22:00:28,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741868_1044 (size=5165) 2024-11-21T22:00:28,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741868_1044 (size=5165) 2024-11-21T22:00:28,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741868_1044 (size=5165) 2024-11-21T22:00:28,174 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/table/bff417f50e98435da30513b0498379c0 2024-11-21T22:00:28,184 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/info/2a765146a1dc4139af777ce21c79fa9b as hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/info/2a765146a1dc4139af777ce21c79fa9b 2024-11-21T22:00:28,194 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/info/2a765146a1dc4139af777ce21c79fa9b, entries=273, sequenceid=64, filesize=35.5 K 2024-11-21T22:00:28,198 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/ns/b4b8572cca8c42e6b789f61aba493ab3 as hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/ns/b4b8572cca8c42e6b789f61aba493ab3 2024-11-21T22:00:28,209 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/ns/b4b8572cca8c42e6b789f61aba493ab3, entries=2, sequenceid=64, filesize=5.0 K 2024-11-21T22:00:28,211 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/table/bff417f50e98435da30513b0498379c0 as hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/table/bff417f50e98435da30513b0498379c0 2024-11-21T22:00:28,226 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/table/bff417f50e98435da30513b0498379c0, entries=2, sequenceid=64, filesize=5.0 K 2024-11-21T22:00:28,230 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 285ms, sequenceid=64, compaction requested=false; wal=null 2024-11-21T22:00:28,231 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T22:00:28,234 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T22:00:28,236 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T22:00:28,236 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T22:00:28,237 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T22:00:28,239 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T22:00:28,242 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-21T22:00:28,244 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70834646, jitterRate=0.05551847815513611}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T22:00:28,244 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T22:00:28,244 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732226427901Writing region info on filesystem at 1732226427901Initializing all the Stores at 1732226427902 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226427902Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226427906 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226427906Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226427906Obtaining lock to block concurrent updates at 1732226427945 (+39 ms)Preparing flush snapshotting stores in 1588230740 at 1732226427945Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732226427949 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732226427949Flushing 1588230740/info: creating writer at 1732226427951 (+2 ms)Flushing 1588230740/info: appending metadata at 1732226428017 (+66 ms)Flushing 1588230740/info: closing flushed file at 1732226428020 (+3 ms)Flushing 1588230740/ns: creating writer at 1732226428092 (+72 ms)Flushing 1588230740/ns: appending metadata at 1732226428114 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732226428114Flushing 1588230740/table: creating writer at 1732226428140 (+26 ms)Flushing 1588230740/table: appending metadata at 1732226428160 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732226428161 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4baf7c35: reopening flushed file at 1732226428182 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5037b356: reopening flushed file at 1732226428196 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49be0b70: reopening flushed file at 1732226428209 (+13 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 285ms, sequenceid=64, compaction requested=false; wal=null at 1732226428230 (+21 ms)Cleaning up temporary data from old regions at 1732226428236 (+6 ms)Running coprocessor post-open hooks at 1732226428244 (+8 ms)Region opened successfully at 1732226428244 2024-11-21T22:00:28,246 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732226427867 2024-11-21T22:00:28,249 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T22:00:28,250 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T22:00:28,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,252 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:28,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:28,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:28,265 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:28,265 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,266 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:28,266 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:28,266 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:28,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-21T22:00:28,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 in 553 msec 2024-11-21T22:00:28,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-21T22:00:28,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 751 msec 2024-11-21T22:00:28,274 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 2a884b411602,34991,1732226420916 had 9 regions 2024-11-21T22:00:28,275 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true, isMeta: false 2024-11-21T22:00:28,278 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2a884b411602,34991,1732226420916 WAL count=1, meta=false 2024-11-21T22:00:28,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541}] 2024-11-21T22:00:28,281 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:28,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36681 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-21T22:00:28,454 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541, size=2.7 K (2774bytes) 2024-11-21T22:00:28,454 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541 2024-11-21T22:00:28,454 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541 after 0ms 2024-11-21T22:00:28,458 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541 took 5ms 2024-11-21T22:00:28,461 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for e8aafe78c77f56bf1d0a5753056c1486: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T22:00:28,463 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 15ef09871c0bb53efdb85d5008b7e94c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T22:00:28,465 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d6cc8ee4cfeffaf8625f8fdd19f2e2ff: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T22:00:28,468 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for bb120caa77e7b0578515651fde091e69: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T22:00:28,470 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8f8584ecc213f380a7bf9bef219b6473: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T22:00:28,476 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for b3fe4283fabe9b42395c59e1bedb1cba: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T22:00:28,479 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 30e7bcb84c1a622f86b573b1a579eabb: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T22:00:28,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d58deb362db49650ac0424f413c0d8a1: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T22:00:28,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 95832a57de44abb16e1581ee9057cab8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T22:00:28,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541 so closing down 2024-11-21T22:00:28,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T22:00:28,484 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T22:00:28,485 INFO [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 27 ms; skipped=10; WAL=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-21T22:00:28,486 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541, journal: Splitting hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541, size=2.7 K (2774bytes) at 1732226428454Finishing writing output for hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541 so closing down at 1732226428484 (+30 ms)3 split writer threads finished at 1732226428485 (+1 ms)Processed 10 edits across 0 Regions in 27 ms; skipped=10; WAL=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1732226428486 (+1 ms) 2024-11-21T22:00:28,487 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541 2024-11-21T22:00:28,487 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-21T22:00:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38029 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-21T22:00:28,492 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting/2a884b411602%2C34991%2C1732226420916.1732226422541 to hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/oldWALs 2024-11-21T22:00:28,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-21T22:00:28,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 in 210 msec 2024-11-21T22:00:28,498 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-21T22:00:28,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 in 220 msec 2024-11-21T22:00:28,504 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,34991,1732226420916-splitting dir is empty, no logs to split. 2024-11-21T22:00:28,504 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2a884b411602,34991,1732226420916 WAL count=0, meta=false 2024-11-21T22:00:28,504 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2a884b411602,34991,1732226420916 WAL splitting is done? wals=0, meta=false 2024-11-21T22:00:28,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN}] 2024-11-21T22:00:28,509 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:28,511 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T22:00:28,513 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:28,514 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T22:00:28,516 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:28,517 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T22:00:28,520 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:28,522 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T22:00:28,523 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:28,525 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T22:00:28,528 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:28,530 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T22:00:28,533 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T22:00:28,535 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:28,537 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T22:00:28,539 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:28,540 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T22:00:28,602 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-21T22:00:28,662 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T22:00:28,662 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T22:00:28,663 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T22:00:28,663 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T22:00:28,663 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T22:00:28,663 INFO [2a884b411602:38029 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T22:00:28,663 INFO [2a884b411602:38029 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T22:00:28,663 DEBUG [2a884b411602:38029 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T22:00:28,663 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,664 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,665 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,665 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2a884b411602:34991 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T22:00:28,666 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, regionLocation=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,666 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:28,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1, error=java.net.ConnectException: Call to address=2a884b411602:34991 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 2024-11-21T22:00:28,667 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:28,668 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=bb120caa77e7b0578515651fde091e69, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:28,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-21T22:00:28,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1 from cache 2024-11-21T22:00:28,669 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, regionLocation=2a884b411602,45209,1732226420988 2024-11-21T22:00:28,669 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T22:00:28,669 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2a884b411602,36681,1732226420789, seqNum=-1] 2024-11-21T22:00:28,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 2a884b411602:34991 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 2024-11-21T22:00:28,670 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T22:00:28,671 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2a884b411602:34991 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T22:00:28,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2a884b411602,36681,1732226420789, seqNum=-1, error=java.net.ConnectException: Call to address=2a884b411602:34991 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 2024-11-21T22:00:28,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2a884b411602,36681,1732226420789, seqNum=-1, error=java.net.ConnectException: Call to address=2a884b411602:34991 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 2024-11-21T22:00:28,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2a884b411602,36681,1732226420789, seqNum=-1, error=java.net.ConnectException: Call to address=2a884b411602:34991 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 2024-11-21T22:00:28,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2a884b411602,36681,1732226420789, seqNum=-1, error=java.net.ConnectException: Call to address=2a884b411602:34991 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 2024-11-21T22:00:28,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2a884b411602,36681,1732226420789, seqNum=-1, error=java.net.ConnectException: Call to address=2a884b411602:34991 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 2024-11-21T22:00:28,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,34991,1732226420916, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2a884b411602,36681,1732226420789, seqNum=-1, error=java.net.ConnectException: Call to address=2a884b411602:34991 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 2024-11-21T22:00:28,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43229, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T22:00:28,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 2a884b411602:34991 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:34991 2024-11-21T22:00:28,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN because future has completed 2024-11-21T22:00:28,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:28,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN because future has completed 2024-11-21T22:00:28,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN because future has completed 2024-11-21T22:00:28,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:28,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN because future has completed 2024-11-21T22:00:28,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN because future has completed 2024-11-21T22:00:28,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN because future has completed 2024-11-21T22:00:28,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN because future has completed 2024-11-21T22:00:28,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN because future has completed 2024-11-21T22:00:28,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:28,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:28,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:28,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:28,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,36681,1732226420789}] 2024-11-21T22:00:28,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb120caa77e7b0578515651fde091e69, server=2a884b411602,45209,1732226420988}] 2024-11-21T22:00:28,837 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:28,837 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => b3fe4283fabe9b42395c59e1bedb1cba, NAME => 'test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-21T22:00:28,837 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,837 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:28,838 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,838 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,839 INFO [StoreOpener-b3fe4283fabe9b42395c59e1bedb1cba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,841 INFO [StoreOpener-b3fe4283fabe9b42395c59e1bedb1cba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3fe4283fabe9b42395c59e1bedb1cba columnFamilyName family 2024-11-21T22:00:28,841 DEBUG [StoreOpener-b3fe4283fabe9b42395c59e1bedb1cba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:28,842 INFO [StoreOpener-b3fe4283fabe9b42395c59e1bedb1cba-1 {}] regionserver.HStore(327): Store=b3fe4283fabe9b42395c59e1bedb1cba/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:28,843 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,844 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,846 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,847 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,847 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,850 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,851 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened b3fe4283fabe9b42395c59e1bedb1cba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75110778, jitterRate=0.11923781037330627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:28,852 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:28,852 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for b3fe4283fabe9b42395c59e1bedb1cba: Running coprocessor pre-open hook at 1732226428838Writing region info on filesystem at 1732226428838Initializing all the Stores at 1732226428839 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226428839Cleaning up temporary data from old regions at 1732226428847 (+8 ms)Running coprocessor post-open hooks at 1732226428852 (+5 ms)Region opened successfully at 1732226428852 2024-11-21T22:00:28,853 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba., pid=77, masterSystemTime=1732226428829 2024-11-21T22:00:28,857 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:28,857 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:28,857 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:28,857 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => d6cc8ee4cfeffaf8625f8fdd19f2e2ff, NAME => 'test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-21T22:00:28,858 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,858 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:28,858 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,858 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,859 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-21T22:00:28,859 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:28,859 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:28,867 INFO [StoreOpener-d6cc8ee4cfeffaf8625f8fdd19f2e2ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,872 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-21T22:00:28,875 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:28,875 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb120caa77e7b0578515651fde091e69, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:28,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:28,875 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb120caa77e7b0578515651fde091e69, server=2a884b411602,45209,1732226420988 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:28,876 INFO [StoreOpener-d6cc8ee4cfeffaf8625f8fdd19f2e2ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cc8ee4cfeffaf8625f8fdd19f2e2ff columnFamilyName family 2024-11-21T22:00:28,876 DEBUG [StoreOpener-d6cc8ee4cfeffaf8625f8fdd19f2e2ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:28,877 INFO [StoreOpener-d6cc8ee4cfeffaf8625f8fdd19f2e2ff-1 {}] regionserver.HStore(327): Store=d6cc8ee4cfeffaf8625f8fdd19f2e2ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:28,877 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,879 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,881 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,882 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,882 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,885 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,886 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened d6cc8ee4cfeffaf8625f8fdd19f2e2ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73886961, jitterRate=0.10100151598453522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:28,887 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:28,887 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for d6cc8ee4cfeffaf8625f8fdd19f2e2ff: Running coprocessor pre-open hook at 1732226428858Writing region info on filesystem at 1732226428858Initializing all the Stores at 1732226428864 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226428864Cleaning up temporary data from old regions at 1732226428882 (+18 ms)Running coprocessor post-open hooks at 1732226428887 (+5 ms)Region opened successfully at 1732226428887 2024-11-21T22:00:28,888 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff., pid=75, masterSystemTime=1732226428829 2024-11-21T22:00:28,891 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:28,891 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:28,891 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:28,891 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 15ef09871c0bb53efdb85d5008b7e94c, NAME => 'test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-21T22:00:28,891 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,892 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:28,892 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,892 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,895 INFO [StoreOpener-15ef09871c0bb53efdb85d5008b7e94c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,897 INFO [StoreOpener-15ef09871c0bb53efdb85d5008b7e94c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15ef09871c0bb53efdb85d5008b7e94c columnFamilyName family 2024-11-21T22:00:28,898 DEBUG [StoreOpener-15ef09871c0bb53efdb85d5008b7e94c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:28,900 INFO [StoreOpener-15ef09871c0bb53efdb85d5008b7e94c-1 {}] regionserver.HStore(327): Store=15ef09871c0bb53efdb85d5008b7e94c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:28,900 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,901 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,904 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,904 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,904 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,907 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,909 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 15ef09871c0bb53efdb85d5008b7e94c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62877587, jitterRate=-0.06305094063282013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:28,909 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:28,909 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 15ef09871c0bb53efdb85d5008b7e94c: Running coprocessor pre-open hook at 1732226428892Writing region info on filesystem at 1732226428892Initializing all the Stores at 1732226428895 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226428895Cleaning up temporary data from old regions at 1732226428904 (+9 ms)Running coprocessor post-open hooks at 1732226428909 (+5 ms)Region opened successfully at 1732226428909 2024-11-21T22:00:28,910 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c., pid=78, masterSystemTime=1732226428829 2024-11-21T22:00:28,914 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:28,914 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:28,942 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:28,942 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 30e7bcb84c1a622f86b573b1a579eabb, NAME => 'test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-21T22:00:28,942 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,942 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:28,942 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,942 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,946 INFO [StoreOpener-30e7bcb84c1a622f86b573b1a579eabb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,947 INFO [StoreOpener-30e7bcb84c1a622f86b573b1a579eabb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30e7bcb84c1a622f86b573b1a579eabb columnFamilyName family 2024-11-21T22:00:28,947 DEBUG [StoreOpener-30e7bcb84c1a622f86b573b1a579eabb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:28,948 INFO [StoreOpener-30e7bcb84c1a622f86b573b1a579eabb-1 {}] regionserver.HStore(327): Store=30e7bcb84c1a622f86b573b1a579eabb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:28,948 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,950 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,951 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,952 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,952 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,954 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,956 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 30e7bcb84c1a622f86b573b1a579eabb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74032703, jitterRate=0.10317324101924896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:28,956 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:28,956 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 30e7bcb84c1a622f86b573b1a579eabb: Running coprocessor pre-open hook at 1732226428943Writing region info on filesystem at 1732226428943Initializing all the Stores at 1732226428945 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226428945Cleaning up temporary data from old regions at 1732226428952 (+7 ms)Running coprocessor post-open hooks at 1732226428956 (+4 ms)Region opened successfully at 1732226428956 2024-11-21T22:00:28,957 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb., pid=80, masterSystemTime=1732226428936 2024-11-21T22:00:28,960 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:28,960 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:28,960 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. 2024-11-21T22:00:28,960 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => e8aafe78c77f56bf1d0a5753056c1486, NAME => 'test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-21T22:00:28,961 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,961 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:28,961 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,961 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,963 INFO [StoreOpener-e8aafe78c77f56bf1d0a5753056c1486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,964 INFO [StoreOpener-e8aafe78c77f56bf1d0a5753056c1486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8aafe78c77f56bf1d0a5753056c1486 columnFamilyName family 2024-11-21T22:00:28,964 DEBUG [StoreOpener-e8aafe78c77f56bf1d0a5753056c1486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:28,965 INFO [StoreOpener-e8aafe78c77f56bf1d0a5753056c1486-1 {}] regionserver.HStore(327): Store=e8aafe78c77f56bf1d0a5753056c1486/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:28,965 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,966 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,967 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,968 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,968 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,970 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,972 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened e8aafe78c77f56bf1d0a5753056c1486; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63591915, jitterRate=-0.05240662395954132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:28,972 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:28,972 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for e8aafe78c77f56bf1d0a5753056c1486: Running coprocessor pre-open hook at 1732226428961Writing region info on filesystem at 1732226428961Initializing all the Stores at 1732226428963 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226428963Cleaning up temporary data from old regions at 1732226428968 (+5 ms)Running coprocessor post-open hooks at 1732226428972 (+4 ms)Region opened successfully at 1732226428972 2024-11-21T22:00:28,972 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:28,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:28,972 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:28,973 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:28,973 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486., pid=79, masterSystemTime=1732226428936 2024-11-21T22:00:28,974 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:28,974 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:28,974 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:28,974 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:28,974 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:28,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 4 msec 2024-11-21T22:00:28,986 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. 2024-11-21T22:00:28,986 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => bb120caa77e7b0578515651fde091e69, NAME => 'test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-21T22:00:28,986 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:28,986 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:28,987 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:28,987 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:28,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:28,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:28,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:28,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:28,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:28,991 INFO [StoreOpener-bb120caa77e7b0578515651fde091e69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:28,991 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:28,992 INFO [StoreOpener-bb120caa77e7b0578515651fde091e69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb120caa77e7b0578515651fde091e69 columnFamilyName family 2024-11-21T22:00:28,992 DEBUG [StoreOpener-bb120caa77e7b0578515651fde091e69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:28,993 INFO [StoreOpener-bb120caa77e7b0578515651fde091e69-1 {}] regionserver.HStore(327): Store=bb120caa77e7b0578515651fde091e69/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:28,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:28,993 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:28,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:28,995 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:28,997 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:28,997 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:28,997 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:28,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:28,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:28,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:28,999 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "2a884b411602" port: 36681 start_code: 1732226420789 } transition { transition_code: OPENED region_info { region_id: 1732226423403 table_name { namespace: "default" qualifier: "test" } start_key: "ccc" end_key: "ddd" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732226421103 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,000 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:29,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb120caa77e7b0578515651fde091e69, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,002 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,002 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,002 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened bb120caa77e7b0578515651fde091e69; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59178581, jitterRate=-0.11817042529582977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,002 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,002 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb120caa77e7b0578515651fde091e69 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,002 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,002 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for bb120caa77e7b0578515651fde091e69: Running coprocessor pre-open hook at 1732226428987Writing region info on filesystem at 1732226428987Initializing all the Stores at 1732226428988 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226428988Cleaning up temporary data from old regions at 1732226428997 (+9 ms)Running coprocessor post-open hooks at 1732226429002 (+5 ms)Region opened successfully at 1732226429002 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb120caa77e7b0578515651fde091e69, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,003 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69., pid=82, masterSystemTime=1732226428980 2024-11-21T22:00:29,003 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 24 msec 2024-11-21T22:00:29,004 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,005 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "2a884b411602" port: 45209 start_code: 1732226420988 } transition { transition_code: OPENED region_info { region_id: 1732226423403 table_name { namespace: "default" qualifier: "test" } start_key: "nnn" end_key: "ooo" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 82 initiating_master_active_time: 1732226421103 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-21T22:00:29,006 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,006 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,032 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,035 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,035 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN to state=OFFLINE, location=null, table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff to restore RIT 2024-11-21T22:00:29,035 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459035 2024-11-21T22:00:29,035 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,036 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,037 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,042 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,042 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,042 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,36681,1732226420789 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,076 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,076 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489076 2024-11-21T22:00:29,076 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549076 2024-11-21T22:00:29,076 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 40 msec 2024-11-21T22:00:29,076 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,076 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,076 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,076 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,077 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,077 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,077 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,077 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,079 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T22:00:29,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,085 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,093 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,093 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,093 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,093 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,093 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d58deb362db49650ac0424f413c0d8a1, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-21T22:00:29,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,095 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,095 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,124 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,126 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,126 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,126 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,126 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,126 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,126 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN to state=OFFLINE, location=null, table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff to restore RIT 2024-11-21T22:00:29,128 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459128 2024-11-21T22:00:29,128 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,128 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,131 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,133 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,135 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,136 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,36681,1732226420789 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,152 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489152 2024-11-21T22:00:29,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549152 2024-11-21T22:00:29,152 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-21T22:00:29,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,152 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,152 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,153 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,153 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,153 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,154 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T22:00:29,158 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e7bcb84c1a622f86b573b1a579eabb, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,167 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-21T22:00:29,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,171 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,171 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,207 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,207 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,207 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,207 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,208 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,208 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,208 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,208 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,208 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,208 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,208 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,208 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN to state=OFFLINE, location=null, table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff to restore RIT 2024-11-21T22:00:29,209 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459209 2024-11-21T22:00:29,209 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,209 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,212 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,212 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,212 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489225 2024-11-21T22:00:29,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549225 2024-11-21T22:00:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-21T22:00:29,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,225 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,226 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,226 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,226 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,241 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,241 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,241 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,241 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,241 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8aafe78c77f56bf1d0a5753056c1486, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,242 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-21T22:00:29,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,244 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,245 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,253 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,262 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,263 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN to state=OFFLINE, location=null, table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff to restore RIT 2024-11-21T22:00:29,264 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459264 2024-11-21T22:00:29,264 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,264 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,266 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,271 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,271 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,271 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,278 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,278 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489278 2024-11-21T22:00:29,279 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549279 2024-11-21T22:00:29,279 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-21T22:00:29,279 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,279 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,279 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,279 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,279 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,280 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,281 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,286 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,286 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,286 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,286 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,286 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,290 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,290 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,290 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15ef09871c0bb53efdb85d5008b7e94c, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,291 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-21T22:00:29,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,292 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,293 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,332 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN to state=OFFLINE, location=null, table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff to restore RIT 2024-11-21T22:00:29,334 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459334 2024-11-21T22:00:29,334 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,334 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,335 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,336 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,336 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,336 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489347 2024-11-21T22:00:29,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549347 2024-11-21T22:00:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-21T22:00:29,347 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,347 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,347 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,347 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,348 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,349 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,354 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,354 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,357 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,359 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,359 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,359 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,359 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,359 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-21T22:00:29,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,360 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,360 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,364 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,366 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,373 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN to state=OFFLINE, location=null, table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff to restore RIT 2024-11-21T22:00:29,375 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459375 2024-11-21T22:00:29,375 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,376 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,377 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,378 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,378 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,378 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,387 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,387 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489387 2024-11-21T22:00:29,387 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549387 2024-11-21T22:00:29,387 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-21T22:00:29,387 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,387 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,387 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,387 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,388 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,388 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,388 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,388 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,389 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,393 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,394 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,399 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f8584ecc213f380a7bf9bef219b6473, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,399 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-21T22:00:29,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,400 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,400 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,406 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,407 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,407 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,407 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN to state=OFFLINE, location=null, table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff to restore RIT 2024-11-21T22:00:29,409 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459409 2024-11-21T22:00:29,409 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,409 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,410 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,411 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,411 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,411 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,420 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,420 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489420 2024-11-21T22:00:29,420 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549420 2024-11-21T22:00:29,420 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-21T22:00:29,420 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,420 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,420 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,421 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,421 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,421 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,421 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,421 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,426 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,431 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,431 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,431 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,431 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,431 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,431 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,432 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,432 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,432 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,432 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,432 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,432 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,432 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6cc8ee4cfeffaf8625f8fdd19f2e2ff, server=2a884b411602,45209,1732226420988 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,432 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-21T22:00:29,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,433 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,433 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,440 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,442 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,442 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,442 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,442 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,442 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN to state=OFFLINE, location=null, table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff to restore RIT 2024-11-21T22:00:29,442 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459442 2024-11-21T22:00:29,442 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,442 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,444 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,445 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,445 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,445 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,453 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,453 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489453 2024-11-21T22:00:29,454 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549454 2024-11-21T22:00:29,454 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-21T22:00:29,454 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,454 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,454 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,454 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,455 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,455 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,455 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,456 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,456 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,465 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,465 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,465 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,465 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,465 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,466 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,466 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,466 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,466 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,466 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,466 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,466 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,466 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN 2024-11-21T22:00:29,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-21T22:00:29,467 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,467 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,468 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,476 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,476 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,479 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,479 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,479 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,479 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,480 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, ASSIGN to state=OFFLINE, location=null, table=test, region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff to restore RIT 2024-11-21T22:00:29,481 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459481 2024-11-21T22:00:29,482 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,482 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,483 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,490 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,490 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,490 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,491 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,491 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489491 2024-11-21T22:00:29,491 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549491 2024-11-21T22:00:29,491 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T22:00:29,492 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,492 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,492 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,492 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,492 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,492 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,492 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,492 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,493 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,498 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,498 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,498 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,498 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,499 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,499 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,499 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,499 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,500 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,500 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,500 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,500 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,500 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,500 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,500 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,500 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN 2024-11-21T22:00:29,500 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-21T22:00:29,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,501 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,501 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,512 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,514 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e8aafe78c77f56bf1d0a5753056c1486, ASSIGN to state=OFFLINE, location=null, table=test, region=e8aafe78c77f56bf1d0a5753056c1486 to restore RIT 2024-11-21T22:00:29,515 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459515 2024-11-21T22:00:29,515 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,515 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,517 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,518 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,518 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,518 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,525 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489525 2024-11-21T22:00:29,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549525 2024-11-21T22:00:29,525 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-21T22:00:29,525 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,525 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,525 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,525 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,526 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,526 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,526 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,527 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,531 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,531 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,535 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,535 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,535 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN 2024-11-21T22:00:29,535 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-21T22:00:29,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,536 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,536 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,546 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,546 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,546 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,546 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,546 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,546 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,547 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,547 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,547 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,547 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8f8584ecc213f380a7bf9bef219b6473, ASSIGN to state=OFFLINE, location=null, table=test, region=8f8584ecc213f380a7bf9bef219b6473 to restore RIT 2024-11-21T22:00:29,547 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459547 2024-11-21T22:00:29,547 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,548 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,549 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,551 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,552 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,552 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,560 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,560 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489560 2024-11-21T22:00:29,560 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549560 2024-11-21T22:00:29,560 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-21T22:00:29,560 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,560 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,560 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,560 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,561 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,561 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,561 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,561 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,562 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,569 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN 2024-11-21T22:00:29,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-21T22:00:29,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,570 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,570 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,581 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,582 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,582 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3fe4283fabe9b42395c59e1bedb1cba, ASSIGN to state=OFFLINE, location=null, table=test, region=b3fe4283fabe9b42395c59e1bedb1cba to restore RIT 2024-11-21T22:00:29,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,587 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459587 2024-11-21T22:00:29,587 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,587 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,588 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,591 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,596 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,596 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489596 2024-11-21T22:00:29,596 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549596 2024-11-21T22:00:29,596 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T22:00:29,596 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,596 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,597 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,597 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,597 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,597 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,597 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,598 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,604 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,604 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,604 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,604 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN 2024-11-21T22:00:29,604 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T22:00:29,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,607 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,608 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,615 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,617 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,617 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,617 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=15ef09871c0bb53efdb85d5008b7e94c, ASSIGN to state=OFFLINE, location=null, table=test, region=15ef09871c0bb53efdb85d5008b7e94c to restore RIT 2024-11-21T22:00:29,620 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459620 2024-11-21T22:00:29,620 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,620 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,622 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,625 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,626 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,626 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,631 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,631 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489631 2024-11-21T22:00:29,631 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549631 2024-11-21T22:00:29,631 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-21T22:00:29,631 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,631 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,631 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,631 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,632 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,632 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,632 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,632 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,633 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,640 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,640 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,640 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,640 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN 2024-11-21T22:00:29,641 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-21T22:00:29,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,643 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,643 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,654 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,655 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,655 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,656 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb120caa77e7b0578515651fde091e69, ASSIGN to state=OFFLINE, location=null, table=test, region=bb120caa77e7b0578515651fde091e69 to restore RIT 2024-11-21T22:00:29,658 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459658 2024-11-21T22:00:29,659 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,659 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,660 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,661 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,661 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,662 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,668 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,668 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489668 2024-11-21T22:00:29,668 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549668 2024-11-21T22:00:29,668 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T22:00:29,668 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,668 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,668 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,668 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,668 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,668 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,669 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,669 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,670 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN 2024-11-21T22:00:29,676 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T22:00:29,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,677 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,677 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,686 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,686 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,707 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,708 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,708 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=30e7bcb84c1a622f86b573b1a579eabb, ASSIGN to state=OFFLINE, location=null, table=test, region=30e7bcb84c1a622f86b573b1a579eabb to restore RIT 2024-11-21T22:00:29,709 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459709 2024-11-21T22:00:29,709 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,709 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,710 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,713 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,713 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,714 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,716 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,716 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489716 2024-11-21T22:00:29,716 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549716 2024-11-21T22:00:29,716 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T22:00:29,716 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,716 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,716 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,716 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,716 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,716 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,716 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,717 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,717 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,720 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,720 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,720 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,721 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,722 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,722 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,722 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN 2024-11-21T22:00:29,722 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T22:00:29,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,723 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,723 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,740 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,740 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,740 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,740 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,741 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d58deb362db49650ac0424f413c0d8a1, ASSIGN to state=OFFLINE, location=null, table=test, region=d58deb362db49650ac0424f413c0d8a1 to restore RIT 2024-11-21T22:00:29,742 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459742 2024-11-21T22:00:29,742 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,742 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,743 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,746 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,746 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,746 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,750 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,750 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489750 2024-11-21T22:00:29,750 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549750 2024-11-21T22:00:29,750 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T22:00:29,750 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,750 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,750 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,751 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,751 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,751 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,751 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,751 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,752 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,756 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,757 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T22:00:29,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,759 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,760 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,771 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,775 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459774 2024-11-21T22:00:29,775 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,775 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,776 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,781 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,781 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,781 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,785 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489785 2024-11-21T22:00:29,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549785 2024-11-21T22:00:29,785 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-21T22:00:29,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,785 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,785 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,786 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,786 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,786 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,790 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,790 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.1732226422541, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,792 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T22:00:29,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,793 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,793 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,804 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,805 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459805 2024-11-21T22:00:29,805 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,805 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,809 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,811 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,811 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,811 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,814 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,814 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489814 2024-11-21T22:00:29,814 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549814 2024-11-21T22:00:29,814 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T22:00:29,814 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,814 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,814 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,815 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,815 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,815 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,815 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,815 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,816 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T22:00:29,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,820 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T22:00:29,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,821 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,821 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,831 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,832 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,832 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,832 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,834 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459834 2024-11-21T22:00:29,834 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,834 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,835 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,839 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,843 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,843 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489843 2024-11-21T22:00:29,843 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549843 2024-11-21T22:00:29,843 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T22:00:29,843 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,843 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,843 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,843 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,844 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,844 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,844 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,844 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,845 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T22:00:29,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T22:00:29,848 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,849 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,849 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,849 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T22:00:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,853 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,853 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,862 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,863 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,864 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459864 2024-11-21T22:00:29,864 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,864 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,865 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,866 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,867 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,867 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,873 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,873 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489873 2024-11-21T22:00:29,873 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549873 2024-11-21T22:00:29,873 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T22:00:29,873 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,873 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,873 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,873 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,874 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,874 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,874 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,874 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,875 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T22:00:29,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,879 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,879 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,879 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,879 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T22:00:29,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,883 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,884 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,890 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,890 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,890 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,890 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,891 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459891 2024-11-21T22:00:29,891 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,891 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,894 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,894 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,894 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,901 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,901 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489901 2024-11-21T22:00:29,901 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549901 2024-11-21T22:00:29,901 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T22:00:29,901 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,901 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,901 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,901 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,901 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,901 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,901 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,902 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,902 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T22:00:29,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 2024-11-21T22:00:29,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,905 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 2a884b411602%2C34991%2C1732226420916.meta.1732226422982.meta, worker=2a884b411602,36681,1732226420789 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T22:00:29,905 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T22:00:29,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,906 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,906 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,912 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,913 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38029 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,915 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,916 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459916 2024-11-21T22:00:29,916 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,917 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,918 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,921 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T22:00:29,921 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,921 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,924 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,924 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489924 2024-11-21T22:00:29,924 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549924 2024-11-21T22:00:29,924 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T22:00:29,924 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T22:00:29,924 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,924 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:29,924 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:29,924 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T22:00:29,924 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T22:00:29,924 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T22:00:29,925 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T22:00:29,925 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T22:00:29,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T22:00:29,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T22:00:29,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T22:00:29,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95832a57de44abb16e1581ee9057cab8, REOPEN/MOVE 2024-11-21T22:00:29,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true 2024-11-21T22:00:29,927 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T22:00:29,928 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T22:00:29,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=65 2024-11-21T22:00:29,928 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2a884b411602,36681,1732226420789, table=hbase:meta, region=1588230740 2024-11-21T22:00:29,928 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,36681,1732226420789, state=OPEN 2024-11-21T22:00:29,937 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T22:00:29,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T22:00:29,938 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226459938 2024-11-21T22:00:29,938 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T22:00:29,938 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T22:00:29,939 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T22:00:29,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3fe4283fabe9b42395c59e1bedb1cba, server=2a884b411602,45209,1732226420988:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2a884b411602,34991,1732226420916, splitWal=true, meta=true exec-time=2.7740 sec 2024-11-21T22:00:29,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f98ec44fefd547dcab3f48a26fc1df58, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5737853cedadc75f4ba670c884671cc8, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8aafe78c77f56bf1d0a5753056c1486, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6cc8ee4cfeffaf8625f8fdd19f2e2ff, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37682a4b3efbf9497991cc6bf0d15002, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=063f9af8e6526f98d7dbe9567a19314c, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30e7bcb84c1a622f86b573b1a579eabb, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc4fa100e1a67451c5548a58b0b04f47, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46ca3887dfa777a1bf18ad3f42c5ddfb, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95832a57de44abb16e1581ee9057cab8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=5 2024-11-21T22:00:29,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ad08ad34efe19c87e38729ba3c5e49f, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=679d18a37ac81fe200738b66f744f5cf, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e151130b05e8b6cf9cbcab7d52f2339d, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb120caa77e7b0578515651fde091e69, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5115225b7a59d832f569746a9b524ee8, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d58deb362db49650ac0424f413c0d8a1, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a273c548a9d111239f960bce54c80e, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fe395b08c7826341bd5d0e43b4c993a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3fe4283fabe9b42395c59e1bedb1cba, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aff672d78bfa1216741748dbf2072a49, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9442f3a6c1cf81c461647906e5c8134a, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f8584ecc213f380a7bf9bef219b6473, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50e71a63bd2106ad870772d459d21d6b, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a3d82d7cb61c550b508756346794599, regionState=OPEN, lastHost=2a884b411602,45209,1732226420988, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d822a89e8f6f1e186226929ffc5148ff, regionState=OPEN, lastHost=2a884b411602,36681,1732226420789, regionLocation=2a884b411602,36681,1732226420789, openSeqNum=2 2024-11-21T22:00:29,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15ef09871c0bb53efdb85d5008b7e94c, regionState=OPENING, lastHost=2a884b411602,34991,1732226420916, regionLocation=2a884b411602,45209,1732226420988, openSeqNum=2 2024-11-21T22:00:29,947 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T22:00:29,947 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226489947 2024-11-21T22:00:29,947 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226549947 2024-11-21T22:00:29,947 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T22:00:29,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=350 (was 383), OpenFileDescriptor=626 (was 647), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=4122 (was 4369) 2024-11-21T22:00:29,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T22:00:29,966 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T22:00:29,967 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T22:00:29,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:29,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:29,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T22:00:29,967 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T22:00:29,967 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1436097278, stopped=false 2024-11-21T22:00:29,968 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2a884b411602,38029,1732226420059 2024-11-21T22:00:29,979 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T22:00:29,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T22:00:29,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T22:00:29,979 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:29,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:29,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:29,979 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:29,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T22:00:29,980 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T22:00:29,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T22:00:29,980 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T22:00:29,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T22:00:29,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:29,980 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2a884b411602,36681,1732226420789' ***** 2024-11-21T22:00:29,980 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T22:00:29,980 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2a884b411602,45209,1732226420988' ***** 2024-11-21T22:00:29,980 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T22:00:29,981 INFO [RS:2;2a884b411602:45209 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T22:00:29,981 INFO [RS:0;2a884b411602:36681 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T22:00:29,981 INFO [RS:2;2a884b411602:45209 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T22:00:29,981 INFO [RS:2;2a884b411602:45209 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T22:00:29,981 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T22:00:29,981 INFO [RS:0;2a884b411602:36681 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T22:00:29,981 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for 46ca3887dfa777a1bf18ad3f42c5ddfb 2024-11-21T22:00:29,981 INFO [RS:0;2a884b411602:36681 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T22:00:29,981 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for 9ad08ad34efe19c87e38729ba3c5e49f 2024-11-21T22:00:29,981 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T22:00:29,982 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for 95832a57de44abb16e1581ee9057cab8 2024-11-21T22:00:29,982 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for 9442f3a6c1cf81c461647906e5c8134a 2024-11-21T22:00:29,982 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for 063f9af8e6526f98d7dbe9567a19314c 2024-11-21T22:00:29,982 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for 5115225b7a59d832f569746a9b524ee8 2024-11-21T22:00:29,982 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for 679d18a37ac81fe200738b66f744f5cf 2024-11-21T22:00:29,982 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46ca3887dfa777a1bf18ad3f42c5ddfb, disabling compactions & flushes 2024-11-21T22:00:29,982 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for 15ef09871c0bb53efdb85d5008b7e94c 2024-11-21T22:00:29,982 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:29,983 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for 6a3d82d7cb61c550b508756346794599 2024-11-21T22:00:29,983 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:29,983 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for b3fe4283fabe9b42395c59e1bedb1cba 2024-11-21T22:00:29,983 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. after waiting 0 ms 2024-11-21T22:00:29,983 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:29,983 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:29,983 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for 6fe395b08c7826341bd5d0e43b4c993a 2024-11-21T22:00:29,983 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for d6cc8ee4cfeffaf8625f8fdd19f2e2ff 2024-11-21T22:00:29,983 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for 30e7bcb84c1a622f86b573b1a579eabb 2024-11-21T22:00:29,983 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for 5737853cedadc75f4ba670c884671cc8 2024-11-21T22:00:29,983 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(959): stopping server 2a884b411602,45209,1732226420988 2024-11-21T22:00:29,983 INFO [RS:2;2a884b411602:45209 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T22:00:29,983 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for e151130b05e8b6cf9cbcab7d52f2339d 2024-11-21T22:00:29,983 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9ad08ad34efe19c87e38729ba3c5e49f, disabling compactions & flushes 2024-11-21T22:00:29,983 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:29,983 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:29,983 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for aff672d78bfa1216741748dbf2072a49 2024-11-21T22:00:29,983 INFO [RS:2;2a884b411602:45209 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2a884b411602:45209. 2024-11-21T22:00:29,983 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. after waiting 0 ms 2024-11-21T22:00:29,983 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for d822a89e8f6f1e186226929ffc5148ff 2024-11-21T22:00:29,983 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:29,983 DEBUG [RS:2;2a884b411602:45209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T22:00:29,983 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for 17a273c548a9d111239f960bce54c80e 2024-11-21T22:00:29,983 DEBUG [RS:2;2a884b411602:45209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:29,983 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for 37682a4b3efbf9497991cc6bf0d15002 2024-11-21T22:00:29,983 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:29,983 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for 50e71a63bd2106ad870772d459d21d6b 2024-11-21T22:00:29,984 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(959): stopping server 2a884b411602,36681,1732226420789 2024-11-21T22:00:29,984 INFO [RS:0;2a884b411602:36681 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T22:00:29,984 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-21T22:00:29,984 INFO [RS:0;2a884b411602:36681 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2a884b411602:36681. 2024-11-21T22:00:29,984 DEBUG [RS:0;2a884b411602:36681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T22:00:29,984 DEBUG [RS:0;2a884b411602:36681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:29,984 DEBUG [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(1351): Waiting on 063f9af8e6526f98d7dbe9567a19314c, 15ef09871c0bb53efdb85d5008b7e94c, 46ca3887dfa777a1bf18ad3f42c5ddfb, 5115225b7a59d832f569746a9b524ee8, 679d18a37ac81fe200738b66f744f5cf, 6a3d82d7cb61c550b508756346794599, 6fe395b08c7826341bd5d0e43b4c993a, 9442f3a6c1cf81c461647906e5c8134a, 95832a57de44abb16e1581ee9057cab8, b3fe4283fabe9b42395c59e1bedb1cba, bb120caa77e7b0578515651fde091e69, d6cc8ee4cfeffaf8625f8fdd19f2e2ff, f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:29,984 INFO [RS:0;2a884b411602:36681 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T22:00:29,984 INFO [RS:0;2a884b411602:36681 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T22:00:29,984 INFO [RS:0;2a884b411602:36681 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T22:00:29,984 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T22:00:29,985 INFO [regionserver/2a884b411602:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T22:00:29,985 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-21T22:00:29,985 DEBUG [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17a273c548a9d111239f960bce54c80e, 30e7bcb84c1a622f86b573b1a579eabb, 37682a4b3efbf9497991cc6bf0d15002, 50e71a63bd2106ad870772d459d21d6b, 5737853cedadc75f4ba670c884671cc8, 9ad08ad34efe19c87e38729ba3c5e49f, aff672d78bfa1216741748dbf2072a49, cc4fa100e1a67451c5548a58b0b04f47, d822a89e8f6f1e186226929ffc5148ff, e151130b05e8b6cf9cbcab7d52f2339d, e8aafe78c77f56bf1d0a5753056c1486 2024-11-21T22:00:29,985 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T22:00:29,985 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T22:00:29,985 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T22:00:29,985 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T22:00:29,985 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T22:00:29,985 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-11-21T22:00:29,991 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/46ca3887dfa777a1bf18ad3f42c5ddfb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:29,992 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:29,992 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46ca3887dfa777a1bf18ad3f42c5ddfb: Waiting for close lock at 1732226429982Running coprocessor pre-close hooks at 1732226429982Disabling compacts and flushes for region at 1732226429982Disabling writes for close at 1732226429983 (+1 ms)Writing region close event to WAL at 1732226429985 (+2 ms)Running coprocessor post-close hooks at 1732226429992 (+7 ms)Closed at 1732226429992 2024-11-21T22:00:29,992 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732226423403.46ca3887dfa777a1bf18ad3f42c5ddfb. 2024-11-21T22:00:29,993 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 95832a57de44abb16e1581ee9057cab8, disabling compactions & flushes 2024-11-21T22:00:29,993 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:29,993 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:29,993 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/9ad08ad34efe19c87e38729ba3c5e49f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:29,993 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. after waiting 0 ms 2024-11-21T22:00:29,993 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:29,994 INFO [regionserver/2a884b411602:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T22:00:29,994 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:29,994 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9ad08ad34efe19c87e38729ba3c5e49f: Waiting for close lock at 1732226429983Running coprocessor pre-close hooks at 1732226429983Disabling compacts and flushes for region at 1732226429983Disabling writes for close at 1732226429983Writing region close event to WAL at 1732226429985 (+2 ms)Running coprocessor post-close hooks at 1732226429994 (+9 ms)Closed at 1732226429994 2024-11-21T22:00:29,994 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732226423403.9ad08ad34efe19c87e38729ba3c5e49f. 2024-11-21T22:00:29,994 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30e7bcb84c1a622f86b573b1a579eabb, disabling compactions & flushes 2024-11-21T22:00:29,994 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:29,994 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:29,994 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. after waiting 0 ms 2024-11-21T22:00:29,994 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:30,002 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/info/44763ea787bd4452abf42dde73641db5 is 138, key is test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486./info:regioninfo/1732226428666/Put/seqid=0 2024-11-21T22:00:30,003 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/30e7bcb84c1a622f86b573b1a579eabb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,005 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:30,005 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30e7bcb84c1a622f86b573b1a579eabb: Waiting for close lock at 1732226429994Running coprocessor pre-close hooks at 1732226429994Disabling compacts and flushes for region at 1732226429994Disabling writes for close at 1732226429994Writing region close event to WAL at 1732226429999 (+5 ms)Running coprocessor post-close hooks at 1732226430005 (+6 ms)Closed at 1732226430005 2024-11-21T22:00:30,005 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732226423403.30e7bcb84c1a622f86b573b1a579eabb. 2024-11-21T22:00:30,005 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5737853cedadc75f4ba670c884671cc8, disabling compactions & flushes 2024-11-21T22:00:30,005 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:30,005 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:30,005 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. after waiting 0 ms 2024-11-21T22:00:30,005 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:30,005 WARN [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486., aborting... java.io.IOException: Failed to report opened region to master: test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T22:00:30,006 ERROR [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2440): ***** ABORTING region server 2a884b411602,36681,1732226420789: Failed to open region test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ccc,1732226423403.e8aafe78c77f56bf1d0a5753056c1486. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T22:00:30,006 ERROR [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T22:00:30,006 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T22:00:30,007 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T22:00:30,010 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T22:00:30,011 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T22:00:30,011 WARN [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69., aborting... java.io.IOException: Failed to report opened region to master: test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T22:00:30,012 ERROR [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2440): ***** ABORTING region server 2a884b411602,45209,1732226420988: Failed to open region test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,nnn,1732226423403.bb120caa77e7b0578515651fde091e69. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T22:00:30,012 ERROR [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T22:00:30,012 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T22:00:30,012 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T22:00:30,015 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T22:00:30,016 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T22:00:30,021 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 483260416 }, "NonHeapMemoryUsage": { "committed": 164036608, "init": 7667712, "max": -1, "used": 161462616 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2a884b411602", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 36071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 105, "ProcessCallTime_min": 0, "ProcessCallTime_max": 17, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 8, "ProcessCallTime_75th_percentile": 12, "ProcessCallTime_90th_percentile": 15, "ProcessCallTime_95th_percentile": 16, "ProcessCallTime_98th_percentile": 16, "ProcessCallTime_99th_percentile": 16, "ProcessCallTime_99.9th_percentile": 16, "ProcessCallTime_TimeRangeCount_0-1": 105, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 105, "QueueCallTime_min": 0, "QueueCallTime_max": 10, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 7, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 9, "QueueCallTime_98th_percentile": 9, "QueueCallTime_99th_percentile": 9, "QueueCallTime_99.9th_percentile": 9, "QueueCallTime_TimeRangeCount_0-1": 105, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 105, "TotalCallTime_min": 0, "TotalCallTime_max": 17, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 8, "TotalCallTime_75th_percentile": 12, "TotalCallTime_90th_percentile": 15, "TotalCallTime_95th_percentile": 16, "TotalCallTime_98th_percentile": 16, "TotalCallTime_99th_percentile": 16, "TotalCallTime_99.9th_percentile": 16, "TotalCallTime_TimeRangeCount_0-1": 105, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 105, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 651, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 105, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 105, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 312, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 105, "sentBytes": 54788 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2a884b411602", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62211", "tag.serverName": "2a884b411602,34991,1732226420916", "tag.clusterId": "70ad5833-ee99-4eba-8991-80a4feccbd08", "tag.Context": "regionserver", "tag.Hostname": "2a884b411602", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 30111, "memStoreHeapSize": 53936, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732226420916, "averageRegionSize": 3345, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3337, "localBytesRead": 3337, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 17.2, "ByteBuffAllocatorHeapAllocationBytes": 7466, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 79, "totalRowActionRequestCount": 180, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 86, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 60, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 14, "ScanTime_mean": 1, "ScanTime_25th_percentile": 3, "ScanTime_median": 7, "ScanTime_75th_percentile": 10, "ScanTime_90th_percentile": 12, "ScanTime_95th_percentile": 13, "ScanTime_98th_percentile": 13, "ScanTime_99th_percentile": 13, "ScanTime_99.9th_percentile": 13, "ScanTime_TimeRangeCount_0-1": 16, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 60, "Put_min": 1, "Put_max": 7, "Put_mean": 1, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 60, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 10, "PutBatch_max": 10, "PutBatch_mean": 10, "PutBatch_25th_percentile": 10, "PutBatch_median": 10, "PutBatch_75th_percentile": 10, "PutBatch_90th_percentile": 10, "PutBatch_95th_percentile": 10, "PutBatch_98th_percentile": 10, "PutBatch_99th_percentile": 10, "PutBatch_99.9th_percentile": 10, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 1, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4186, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 16, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-21T22:00:30,023 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/95832a57de44abb16e1581ee9057cab8/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-21T22:00:30,026 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 483260416 }, "NonHeapMemoryUsage": { "committed": 164167680, "init": 7667712, "max": -1, "used": 161511712 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2a884b411602", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 36071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 105, "ProcessCallTime_min": 0, "ProcessCallTime_max": 17, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 8, "ProcessCallTime_75th_percentile": 12, "ProcessCallTime_90th_percentile": 15, "ProcessCallTime_95th_percentile": 16, "ProcessCallTime_98th_percentile": 16, "ProcessCallTime_99th_percentile": 16, "ProcessCallTime_99.9th_percentile": 16, "ProcessCallTime_TimeRangeCount_0-1": 105, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 105, "QueueCallTime_min": 0, "QueueCallTime_max": 10, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 7, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 9, "QueueCallTime_98th_percentile": 9, "QueueCallTime_99th_percentile": 9, "QueueCallTime_99.9th_percentile": 9, "QueueCallTime_TimeRangeCount_0-1": 105, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 105, "TotalCallTime_min": 0, "TotalCallTime_max": 17, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 8, "TotalCallTime_75th_percentile": 12, "TotalCallTime_90th_percentile": 15, "TotalCallTime_95th_percentile": 16, "TotalCallTime_98th_percentile": 16, "TotalCallTime_99th_percentile": 16, "TotalCallTime_99.9th_percentile": 16, "TotalCallTime_TimeRangeCount_0-1": 105, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 105, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 651, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 105, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 105, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 312, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 105, "sentBytes": 54788 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2a884b411602", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62211", "tag.serverName": "2a884b411602,34991,1732226420916", "tag.clusterId": "70ad5833-ee99-4eba-8991-80a4feccbd08", "tag.Context": "regionserver", "tag.Hostname": "2a884b411602", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 30111, "memStoreHeapSize": 53936, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732226420916, "averageRegionSize": 3345, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3337, "localBytesRead": 3337, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 17.2, "ByteBuffAllocatorHeapAllocationBytes": 7466, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 79, "totalRowActionRequestCount": 180, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 86, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 60, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 14, "ScanTime_mean": 1, "ScanTime_25th_percentile": 3, "ScanTime_median": 7, "ScanTime_75th_percentile": 10, "ScanTime_90th_percentile": 12, "ScanTime_95th_percentile": 13, "ScanTime_98th_percentile": 13, "ScanTime_99th_percentile": 13, "ScanTime_99.9th_percentile": 13, "ScanTime_TimeRangeCount_0-1": 16, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 60, "Put_min": 1, "Put_max": 7, "Put_mean": 1, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 60, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 10, "PutBatch_max": 10, "PutBatch_mean": 10, "PutBatch_25th_percentile": 10, "PutBatch_median": 10, "PutBatch_75th_percentile": 10, "PutBatch_90th_percentile": 10, "PutBatch_95th_percentile": 10, "PutBatch_98th_percentile": 10, "PutBatch_99th_percentile": 10, "PutBatch_99.9th_percentile": 10, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 1, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4186, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 16, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-21T22:00:30,027 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:30,027 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 95832a57de44abb16e1581ee9057cab8: Waiting for close lock at 1732226429992Running coprocessor pre-close hooks at 1732226429992Disabling compacts and flushes for region at 1732226429992Disabling writes for close at 1732226429993 (+1 ms)Writing region close event to WAL at 1732226429999 (+6 ms)Running coprocessor post-close hooks at 1732226430024 (+25 ms)Closed at 1732226430025 (+1 ms) 2024-11-21T22:00:30,027 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732226423403.95832a57de44abb16e1581ee9057cab8. 2024-11-21T22:00:30,027 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9442f3a6c1cf81c461647906e5c8134a, disabling compactions & flushes 2024-11-21T22:00:30,027 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:30,027 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:30,025 WARN [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2a884b411602:38029 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2a884b411602:38029 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more 2024-11-21T22:00:30,027 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. after waiting 0 ms 2024-11-21T22:00:30,027 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:30,027 WARN [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2a884b411602:38029 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2a884b411602:38029 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more 2024-11-21T22:00:30,043 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/5737853cedadc75f4ba670c884671cc8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,044 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:30,044 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5737853cedadc75f4ba670c884671cc8: Waiting for close lock at 1732226430005Running coprocessor pre-close hooks at 1732226430005Disabling compacts and flushes for region at 1732226430005Disabling writes for close at 1732226430005Writing region close event to WAL at 1732226430022 (+17 ms)Running coprocessor post-close hooks at 1732226430044 (+22 ms)Closed at 1732226430044 2024-11-21T22:00:30,044 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732226423403.5737853cedadc75f4ba670c884671cc8. 2024-11-21T22:00:30,045 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e151130b05e8b6cf9cbcab7d52f2339d, disabling compactions & flushes 2024-11-21T22:00:30,045 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:30,045 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:30,045 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. after waiting 0 ms 2024-11-21T22:00:30,045 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:30,055 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. 2024-11-21T22:00:30,055 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => d58deb362db49650ac0424f413c0d8a1, NAME => 'test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-21T22:00:30,056 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,056 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:30,056 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,056 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741869_1045 (size=8192) 2024-11-21T22:00:30,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741869_1045 (size=8192) 2024-11-21T22:00:30,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741869_1045 (size=8192) 2024-11-21T22:00:30,059 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/info/44763ea787bd4452abf42dde73641db5 2024-11-21T22:00:30,070 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/.tmp/info/44763ea787bd4452abf42dde73641db5 as hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/info/44763ea787bd4452abf42dde73641db5 2024-11-21T22:00:30,071 INFO [StoreOpener-d58deb362db49650ac0424f413c0d8a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,072 INFO [StoreOpener-d58deb362db49650ac0424f413c0d8a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d58deb362db49650ac0424f413c0d8a1 columnFamilyName family 2024-11-21T22:00:30,072 DEBUG [StoreOpener-d58deb362db49650ac0424f413c0d8a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:30,072 INFO [StoreOpener-d58deb362db49650ac0424f413c0d8a1-1 {}] regionserver.HStore(327): Store=d58deb362db49650ac0424f413c0d8a1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:30,073 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,074 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,076 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,076 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,077 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,084 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,085 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened d58deb362db49650ac0424f413c0d8a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60634090, jitterRate=-0.09648165106773376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:30,086 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/info/44763ea787bd4452abf42dde73641db5, entries=24, sequenceid=75, filesize=8 K 2024-11-21T22:00:30,086 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d58deb362db49650ac0424f413c0d8a1 2024-11-21T22:00:30,086 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for d58deb362db49650ac0424f413c0d8a1: Running coprocessor pre-open hook at 1732226430056Writing region info on filesystem at 1732226430056Initializing all the Stores at 1732226430057 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226430057Cleaning up temporary data from old regions at 1732226430077 (+20 ms)Running coprocessor post-open hooks at 1732226430086 (+9 ms)Region opened successfully at 1732226430086 2024-11-21T22:00:30,087 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=75, compaction requested=false 2024-11-21T22:00:30,087 WARN [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2a884b411602,36681,1732226420789 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T22:00:30,088 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,ppp,1732226423403.d58deb362db49650ac0424f413c0d8a1. and can not recover 2024-11-21T22:00:30,088 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. 2024-11-21T22:00:30,088 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f8584ecc213f380a7bf9bef219b6473, NAME => 'test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-21T22:00:30,088 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,088 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T22:00:30,088 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,089 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,090 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/9442f3a6c1cf81c461647906e5c8134a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,091 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:30,091 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9442f3a6c1cf81c461647906e5c8134a: Waiting for close lock at 1732226430027Running coprocessor pre-close hooks at 1732226430027Disabling compacts and flushes for region at 1732226430027Disabling writes for close at 1732226430027Writing region close event to WAL at 1732226430034 (+7 ms)Running coprocessor post-close hooks at 1732226430091 (+57 ms)Closed at 1732226430091 2024-11-21T22:00:30,091 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/e151130b05e8b6cf9cbcab7d52f2339d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,092 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732226423403.9442f3a6c1cf81c461647906e5c8134a. 2024-11-21T22:00:30,092 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 063f9af8e6526f98d7dbe9567a19314c, disabling compactions & flushes 2024-11-21T22:00:30,092 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:30,092 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:30,092 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. after waiting 0 ms 2024-11-21T22:00:30,092 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:30,092 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:30,092 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e151130b05e8b6cf9cbcab7d52f2339d: Waiting for close lock at 1732226430044Running coprocessor pre-close hooks at 1732226430044Disabling compacts and flushes for region at 1732226430044Disabling writes for close at 1732226430045 (+1 ms)Writing region close event to WAL at 1732226430075 (+30 ms)Running coprocessor post-close hooks at 1732226430092 (+17 ms)Closed at 1732226430092 2024-11-21T22:00:30,093 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732226423403.e151130b05e8b6cf9cbcab7d52f2339d. 2024-11-21T22:00:30,093 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aff672d78bfa1216741748dbf2072a49, disabling compactions & flushes 2024-11-21T22:00:30,093 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:30,093 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:30,093 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. after waiting 0 ms 2024-11-21T22:00:30,093 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:30,098 INFO [StoreOpener-8f8584ecc213f380a7bf9bef219b6473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,099 INFO [StoreOpener-8f8584ecc213f380a7bf9bef219b6473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f8584ecc213f380a7bf9bef219b6473 columnFamilyName family 2024-11-21T22:00:30,100 DEBUG [StoreOpener-8f8584ecc213f380a7bf9bef219b6473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T22:00:30,101 INFO [StoreOpener-8f8584ecc213f380a7bf9bef219b6473-1 {}] regionserver.HStore(327): Store=8f8584ecc213f380a7bf9bef219b6473/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T22:00:30,101 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,102 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,105 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,106 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,106 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,108 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,109 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 8f8584ecc213f380a7bf9bef219b6473; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63203249, jitterRate=-0.058198198676109314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T22:00:30,109 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f8584ecc213f380a7bf9bef219b6473 2024-11-21T22:00:30,110 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 8f8584ecc213f380a7bf9bef219b6473: Running coprocessor pre-open hook at 1732226430089Writing region info on filesystem at 1732226430089Initializing all the Stores at 1732226430090 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226430090Cleaning up temporary data from old regions at 1732226430106 (+16 ms)Running coprocessor post-open hooks at 1732226430109 (+3 ms)Region opened successfully at 1732226430109 2024-11-21T22:00:30,111 WARN [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2a884b411602,36681,1732226420789 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T22:00:30,111 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,vvv,1732226423403.8f8584ecc213f380a7bf9bef219b6473. and can not recover 2024-11-21T22:00:30,112 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/aff672d78bfa1216741748dbf2072a49/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,114 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:30,115 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aff672d78bfa1216741748dbf2072a49: Waiting for close lock at 1732226430093Running coprocessor pre-close hooks at 1732226430093Disabling compacts and flushes for region at 1732226430093Disabling writes for close at 1732226430093Writing region close event to WAL at 1732226430098 (+5 ms)Running coprocessor post-close hooks at 1732226430114 (+16 ms)Closed at 1732226430114 2024-11-21T22:00:30,115 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732226423403.aff672d78bfa1216741748dbf2072a49. 2024-11-21T22:00:30,115 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d822a89e8f6f1e186226929ffc5148ff, disabling compactions & flushes 2024-11-21T22:00:30,115 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:30,115 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:30,115 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. after waiting 0 ms 2024-11-21T22:00:30,115 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:30,121 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/063f9af8e6526f98d7dbe9567a19314c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,123 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:30,123 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 063f9af8e6526f98d7dbe9567a19314c: Waiting for close lock at 1732226430092Running coprocessor pre-close hooks at 1732226430092Disabling compacts and flushes for region at 1732226430092Disabling writes for close at 1732226430092Writing region close event to WAL at 1732226430102 (+10 ms)Running coprocessor post-close hooks at 1732226430123 (+21 ms)Closed at 1732226430123 2024-11-21T22:00:30,123 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732226423403.063f9af8e6526f98d7dbe9567a19314c. 2024-11-21T22:00:30,124 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5115225b7a59d832f569746a9b524ee8, disabling compactions & flushes 2024-11-21T22:00:30,124 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:30,124 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:30,124 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. after waiting 0 ms 2024-11-21T22:00:30,124 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:30,132 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-21T22:00:30,134 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/5115225b7a59d832f569746a9b524ee8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,134 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T22:00:30,134 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T22:00:30,134 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732226429985Running coprocessor pre-close hooks at 1732226429985Disabling compacts and flushes for region at 1732226429985Disabling writes for close at 1732226429985Obtaining lock to block concurrent updates at 1732226429985Preparing flush snapshotting stores in 1588230740 at 1732226429985Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732226429991 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732226429993 (+2 ms)Flushing 1588230740/info: creating writer at 1732226429993Flushing 1588230740/info: appending metadata at 1732226430001 (+8 ms)Flushing 1588230740/info: closing flushed file at 1732226430001Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@759204d: reopening flushed file at 1732226430069 (+68 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=75, compaction requested=false at 1732226430087 (+18 ms)Writing region close event to WAL at 1732226430119 (+32 ms)Running coprocessor post-close hooks at 1732226430134 (+15 ms)Closed at 1732226430134 2024-11-21T22:00:30,134 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T22:00:30,134 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d822a89e8f6f1e186226929ffc5148ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,135 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:30,135 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5115225b7a59d832f569746a9b524ee8: Waiting for close lock at 1732226430124Running coprocessor pre-close hooks at 1732226430124Disabling compacts and flushes for region at 1732226430124Disabling writes for close at 1732226430124Writing region close event to WAL at 1732226430125 (+1 ms)Running coprocessor post-close hooks at 1732226430135 (+10 ms)Closed at 1732226430135 2024-11-21T22:00:30,135 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732226423403.5115225b7a59d832f569746a9b524ee8. 2024-11-21T22:00:30,135 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 679d18a37ac81fe200738b66f744f5cf, disabling compactions & flushes 2024-11-21T22:00:30,135 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:30,135 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:30,135 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:30,135 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d822a89e8f6f1e186226929ffc5148ff: Waiting for close lock at 1732226430115Running coprocessor pre-close hooks at 1732226430115Disabling compacts and flushes for region at 1732226430115Disabling writes for close at 1732226430115Writing region close event to WAL at 1732226430119 (+4 ms)Running coprocessor post-close hooks at 1732226430135 (+16 ms)Closed at 1732226430135 2024-11-21T22:00:30,135 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. after waiting 0 ms 2024-11-21T22:00:30,135 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:30,135 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732226423403.d822a89e8f6f1e186226929ffc5148ff. 2024-11-21T22:00:30,136 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17a273c548a9d111239f960bce54c80e, disabling compactions & flushes 2024-11-21T22:00:30,136 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:30,136 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:30,136 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. after waiting 0 ms 2024-11-21T22:00:30,136 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:30,153 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/679d18a37ac81fe200738b66f744f5cf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,154 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:30,154 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 679d18a37ac81fe200738b66f744f5cf: Waiting for close lock at 1732226430135Running coprocessor pre-close hooks at 1732226430135Disabling compacts and flushes for region at 1732226430135Disabling writes for close at 1732226430135Writing region close event to WAL at 1732226430136 (+1 ms)Running coprocessor post-close hooks at 1732226430154 (+18 ms)Closed at 1732226430154 2024-11-21T22:00:30,154 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732226423403.679d18a37ac81fe200738b66f744f5cf. 2024-11-21T22:00:30,154 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 15ef09871c0bb53efdb85d5008b7e94c, disabling compactions & flushes 2024-11-21T22:00:30,154 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:30,154 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:30,154 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. after waiting 0 ms 2024-11-21T22:00:30,154 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:30,159 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/17a273c548a9d111239f960bce54c80e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,160 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:30,161 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17a273c548a9d111239f960bce54c80e: Waiting for close lock at 1732226430136Running coprocessor pre-close hooks at 1732226430136Disabling compacts and flushes for region at 1732226430136Disabling writes for close at 1732226430136Writing region close event to WAL at 1732226430151 (+15 ms)Running coprocessor post-close hooks at 1732226430160 (+9 ms)Closed at 1732226430160 2024-11-21T22:00:30,161 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732226423403.17a273c548a9d111239f960bce54c80e. 2024-11-21T22:00:30,161 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 37682a4b3efbf9497991cc6bf0d15002, disabling compactions & flushes 2024-11-21T22:00:30,161 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:30,161 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:30,161 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. after waiting 0 ms 2024-11-21T22:00:30,161 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:30,173 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/15ef09871c0bb53efdb85d5008b7e94c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,174 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:30,174 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 15ef09871c0bb53efdb85d5008b7e94c: Waiting for close lock at 1732226430154Running coprocessor pre-close hooks at 1732226430154Disabling compacts and flushes for region at 1732226430154Disabling writes for close at 1732226430154Writing region close event to WAL at 1732226430155 (+1 ms)Running coprocessor post-close hooks at 1732226430174 (+19 ms)Closed at 1732226430174 2024-11-21T22:00:30,174 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732226423403.15ef09871c0bb53efdb85d5008b7e94c. 2024-11-21T22:00:30,174 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6a3d82d7cb61c550b508756346794599, disabling compactions & flushes 2024-11-21T22:00:30,175 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:30,175 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:30,175 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. after waiting 0 ms 2024-11-21T22:00:30,175 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:30,178 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/37682a4b3efbf9497991cc6bf0d15002/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,181 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:30,181 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 37682a4b3efbf9497991cc6bf0d15002: Waiting for close lock at 1732226430161Running coprocessor pre-close hooks at 1732226430161Disabling compacts and flushes for region at 1732226430161Disabling writes for close at 1732226430161Writing region close event to WAL at 1732226430163 (+2 ms)Running coprocessor post-close hooks at 1732226430181 (+18 ms)Closed at 1732226430181 2024-11-21T22:00:30,181 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732226423403.37682a4b3efbf9497991cc6bf0d15002. 2024-11-21T22:00:30,182 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cc4fa100e1a67451c5548a58b0b04f47, disabling compactions & flushes 2024-11-21T22:00:30,182 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:30,182 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:30,182 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. after waiting 0 ms 2024-11-21T22:00:30,182 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:30,184 DEBUG [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(1351): Waiting on 6a3d82d7cb61c550b508756346794599, 6fe395b08c7826341bd5d0e43b4c993a, b3fe4283fabe9b42395c59e1bedb1cba, d6cc8ee4cfeffaf8625f8fdd19f2e2ff, f98ec44fefd547dcab3f48a26fc1df58 2024-11-21T22:00:30,185 DEBUG [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(1351): Waiting on 50e71a63bd2106ad870772d459d21d6b, cc4fa100e1a67451c5548a58b0b04f47 2024-11-21T22:00:30,192 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/6a3d82d7cb61c550b508756346794599/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,193 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:30,193 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6a3d82d7cb61c550b508756346794599: Waiting for close lock at 1732226430174Running coprocessor pre-close hooks at 1732226430174Disabling compacts and flushes for region at 1732226430174Disabling writes for close at 1732226430175 (+1 ms)Writing region close event to WAL at 1732226430179 (+4 ms)Running coprocessor post-close hooks at 1732226430193 (+14 ms)Closed at 1732226430193 2024-11-21T22:00:30,193 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732226423403.6a3d82d7cb61c550b508756346794599. 2024-11-21T22:00:30,193 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b3fe4283fabe9b42395c59e1bedb1cba, disabling compactions & flushes 2024-11-21T22:00:30,193 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:30,194 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:30,194 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. after waiting 0 ms 2024-11-21T22:00:30,194 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:30,198 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/cc4fa100e1a67451c5548a58b0b04f47/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,200 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:30,200 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cc4fa100e1a67451c5548a58b0b04f47: Waiting for close lock at 1732226430182Running coprocessor pre-close hooks at 1732226430182Disabling compacts and flushes for region at 1732226430182Disabling writes for close at 1732226430182Writing region close event to WAL at 1732226430182Running coprocessor post-close hooks at 1732226430200 (+18 ms)Closed at 1732226430200 2024-11-21T22:00:30,200 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732226423403.cc4fa100e1a67451c5548a58b0b04f47. 2024-11-21T22:00:30,200 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 50e71a63bd2106ad870772d459d21d6b, disabling compactions & flushes 2024-11-21T22:00:30,200 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:30,200 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:30,200 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. after waiting 0 ms 2024-11-21T22:00:30,200 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:30,207 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/50e71a63bd2106ad870772d459d21d6b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,209 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:30,209 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 50e71a63bd2106ad870772d459d21d6b: Waiting for close lock at 1732226430200Running coprocessor pre-close hooks at 1732226430200Disabling compacts and flushes for region at 1732226430200Disabling writes for close at 1732226430200Writing region close event to WAL at 1732226430203 (+3 ms)Running coprocessor post-close hooks at 1732226430209 (+6 ms)Closed at 1732226430209 2024-11-21T22:00:30,209 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732226423403.50e71a63bd2106ad870772d459d21d6b. 2024-11-21T22:00:30,210 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/b3fe4283fabe9b42395c59e1bedb1cba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,211 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:30,212 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b3fe4283fabe9b42395c59e1bedb1cba: Waiting for close lock at 1732226430193Running coprocessor pre-close hooks at 1732226430193Disabling compacts and flushes for region at 1732226430193Disabling writes for close at 1732226430194 (+1 ms)Writing region close event to WAL at 1732226430203 (+9 ms)Running coprocessor post-close hooks at 1732226430211 (+8 ms)Closed at 1732226430211 2024-11-21T22:00:30,212 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732226423403.b3fe4283fabe9b42395c59e1bedb1cba. 2024-11-21T22:00:30,212 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f98ec44fefd547dcab3f48a26fc1df58, disabling compactions & flushes 2024-11-21T22:00:30,212 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:30,212 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:30,212 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. after waiting 0 ms 2024-11-21T22:00:30,212 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:30,228 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/f98ec44fefd547dcab3f48a26fc1df58/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,229 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:30,229 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f98ec44fefd547dcab3f48a26fc1df58: Waiting for close lock at 1732226430212Running coprocessor pre-close hooks at 1732226430212Disabling compacts and flushes for region at 1732226430212Disabling writes for close at 1732226430212Writing region close event to WAL at 1732226430213 (+1 ms)Running coprocessor post-close hooks at 1732226430229 (+16 ms)Closed at 1732226430229 2024-11-21T22:00:30,229 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732226423403.f98ec44fefd547dcab3f48a26fc1df58. 2024-11-21T22:00:30,230 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6fe395b08c7826341bd5d0e43b4c993a, disabling compactions & flushes 2024-11-21T22:00:30,230 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:30,230 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:30,230 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. after waiting 0 ms 2024-11-21T22:00:30,230 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:30,237 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/6fe395b08c7826341bd5d0e43b4c993a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,238 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:30,238 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6fe395b08c7826341bd5d0e43b4c993a: Waiting for close lock at 1732226430230Running coprocessor pre-close hooks at 1732226430230Disabling compacts and flushes for region at 1732226430230Disabling writes for close at 1732226430230Writing region close event to WAL at 1732226430230Running coprocessor post-close hooks at 1732226430238 (+8 ms)Closed at 1732226430238 2024-11-21T22:00:30,239 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732226423403.6fe395b08c7826341bd5d0e43b4c993a. 2024-11-21T22:00:30,239 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d6cc8ee4cfeffaf8625f8fdd19f2e2ff, disabling compactions & flushes 2024-11-21T22:00:30,239 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:30,239 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:30,239 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. after waiting 0 ms 2024-11-21T22:00:30,239 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:30,256 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/data/default/test/d6cc8ee4cfeffaf8625f8fdd19f2e2ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T22:00:30,257 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:30,257 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d6cc8ee4cfeffaf8625f8fdd19f2e2ff: Waiting for close lock at 1732226430239Running coprocessor pre-close hooks at 1732226430239Disabling compacts and flushes for region at 1732226430239Disabling writes for close at 1732226430239Writing region close event to WAL at 1732226430243 (+4 ms)Running coprocessor post-close hooks at 1732226430257 (+14 ms)Closed at 1732226430257 2024-11-21T22:00:30,257 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732226423403.d6cc8ee4cfeffaf8625f8fdd19f2e2ff. 2024-11-21T22:00:30,385 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(976): stopping server 2a884b411602,45209,1732226420988; all regions closed. 2024-11-21T22:00:30,385 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(976): stopping server 2a884b411602,36681,1732226420789; all regions closed. 2024-11-21T22:00:30,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741834_1010 (size=6795) 2024-11-21T22:00:30,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741834_1010 (size=6795) 2024-11-21T22:00:30,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741834_1010 (size=6795) 2024-11-21T22:00:30,391 DEBUG [RS:2;2a884b411602:45209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:30,391 INFO [RS:2;2a884b411602:45209 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T22:00:30,391 INFO [RS:2;2a884b411602:45209 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T22:00:30,392 INFO [RS:2;2a884b411602:45209 {}] hbase.ChoreService(370): Chore service for: regionserver/2a884b411602:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T22:00:30,392 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/WALs/2a884b411602,36681,1732226420789/2a884b411602%2C36681%2C1732226420789.meta.1732226427877.meta not finished, retry = 0 2024-11-21T22:00:30,392 INFO [RS:2;2a884b411602:45209 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T22:00:30,392 INFO [RS:2;2a884b411602:45209 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T22:00:30,392 INFO [RS:2;2a884b411602:45209 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T22:00:30,392 INFO [regionserver/2a884b411602:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T22:00:30,392 INFO [RS:2;2a884b411602:45209 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T22:00:30,392 INFO [RS:2;2a884b411602:45209 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45209 2024-11-21T22:00:30,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741865_1041 (size=4201) 2024-11-21T22:00:30,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741865_1041 (size=4201) 2024-11-21T22:00:30,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741865_1041 (size=4201) 2024-11-21T22:00:30,400 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T22:00:30,400 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T22:00:30,401 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-21T22:00:30,401 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-21T22:00:30,402 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T22:00:30,402 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T22:00:30,402 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T22:00:30,402 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T22:00:30,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2a884b411602,45209,1732226420988 2024-11-21T22:00:30,417 INFO [RS:2;2a884b411602:45209 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T22:00:30,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T22:00:30,472 INFO [regionserver/2a884b411602:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T22:00:30,472 INFO [regionserver/2a884b411602:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T22:00:30,475 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2a884b411602,45209,1732226420988] 2024-11-21T22:00:30,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741835_1011 (size=6267) 2024-11-21T22:00:30,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741835_1011 (size=6267) 2024-11-21T22:00:30,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741835_1011 (size=6267) 2024-11-21T22:00:30,498 DEBUG [RS:0;2a884b411602:36681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T22:00:30,498 INFO [RS:0;2a884b411602:36681 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T22:00:30,498 INFO [RS:0;2a884b411602:36681 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T22:00:30,498 INFO [RS:0;2a884b411602:36681 {}] hbase.ChoreService(370): Chore service for: regionserver/2a884b411602:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T22:00:30,498 INFO [RS:0;2a884b411602:36681 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T22:00:30,498 INFO [regionserver/2a884b411602:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T22:00:30,498 INFO [RS:0;2a884b411602:36681 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36681 2024-11-21T22:00:30,534 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2a884b411602,45209,1732226420988 already deleted, retry=false 2024-11-21T22:00:30,534 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2a884b411602,45209,1732226420988 expired; onlineServers=1 2024-11-21T22:00:30,548 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2a884b411602,36681,1732226420789 2024-11-21T22:00:30,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T22:00:30,548 INFO [RS:0;2a884b411602:36681 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T22:00:30,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T22:00:30,576 INFO [RS:2;2a884b411602:45209 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T22:00:30,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x10160b31ce40003, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T22:00:30,576 INFO [RS:2;2a884b411602:45209 {}] regionserver.HRegionServer(1031): Exiting; stopping=2a884b411602,45209,1732226420988; zookeeper connection closed. 2024-11-21T22:00:30,576 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b51d833 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b51d833 2024-11-21T22:00:30,590 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2a884b411602,36681,1732226420789] 2024-11-21T22:00:30,631 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2a884b411602,36681,1732226420789 already deleted, retry=false 2024-11-21T22:00:30,631 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2a884b411602,36681,1732226420789 expired; onlineServers=0 2024-11-21T22:00:30,632 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2a884b411602,38029,1732226420059' ***** 2024-11-21T22:00:30,632 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T22:00:30,632 INFO [M:0;2a884b411602:38029 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T22:00:30,632 INFO [M:0;2a884b411602:38029 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T22:00:30,632 DEBUG [M:0;2a884b411602:38029 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T22:00:30,632 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T22:00:30,632 DEBUG [M:0;2a884b411602:38029 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T22:00:30,632 DEBUG [master/2a884b411602:0:becomeActiveMaster-HFileCleaner.large.0-1732226422075 {}] cleaner.HFileCleaner(306): Exit Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.large.0-1732226422075,5,FailOnTimeoutGroup] 2024-11-21T22:00:30,632 INFO [M:0;2a884b411602:38029 {}] hbase.ChoreService(370): Chore service for: master/2a884b411602:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T22:00:30,632 INFO [M:0;2a884b411602:38029 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T22:00:30,632 DEBUG [master/2a884b411602:0:becomeActiveMaster-HFileCleaner.small.0-1732226422076 {}] cleaner.HFileCleaner(306): Exit Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.small.0-1732226422076,5,FailOnTimeoutGroup] 2024-11-21T22:00:30,633 DEBUG [M:0;2a884b411602:38029 {}] master.HMaster(1795): Stopping service threads 2024-11-21T22:00:30,633 INFO [M:0;2a884b411602:38029 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T22:00:30,633 INFO [M:0;2a884b411602:38029 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T22:00:30,633 INFO [M:0;2a884b411602:38029 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T22:00:30,633 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T22:00:30,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T22:00:30,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T22:00:30,640 DEBUG [M:0;2a884b411602:38029 {}] zookeeper.ZKUtil(347): master:38029-0x10160b31ce40000, quorum=127.0.0.1:62211, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T22:00:30,640 WARN [M:0;2a884b411602:38029 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T22:00:30,641 INFO [M:0;2a884b411602:38029 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/.lastflushedseqids 2024-11-21T22:00:30,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741870_1046 (size=1) 2024-11-21T22:00:30,652 INFO [M:0;2a884b411602:38029 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T22:00:30,653 INFO [M:0;2a884b411602:38029 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T22:00:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741870_1046 (size=1) 2024-11-21T22:00:30,653 DEBUG [M:0;2a884b411602:38029 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T22:00:30,653 INFO [M:0;2a884b411602:38029 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T22:00:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741870_1046 (size=1) 2024-11-21T22:00:30,653 DEBUG [M:0;2a884b411602:38029 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T22:00:30,653 DEBUG [M:0;2a884b411602:38029 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T22:00:30,653 DEBUG [M:0;2a884b411602:38029 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T22:00:30,653 INFO [M:0;2a884b411602:38029 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.47 KB heapSize=315.38 KB 2024-11-21T22:00:30,670 DEBUG [M:0;2a884b411602:38029 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51eb4ea108054a34bd44dcaec8118fe2 is 82, key is hbase:meta,,1/info:regioninfo/1732226428250/Put/seqid=0 2024-11-21T22:00:30,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34171 is added to blk_1073741871_1047 (size=6071) 2024-11-21T22:00:30,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741871_1047 (size=6071) 2024-11-21T22:00:30,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43557 is added to blk_1073741871_1047 (size=6071) 2024-11-21T22:00:30,677 INFO [M:0;2a884b411602:38029 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51eb4ea108054a34bd44dcaec8118fe2 2024-11-21T22:00:30,690 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T22:00:30,690 INFO [RS:0;2a884b411602:36681 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T22:00:30,690 INFO [RS:0;2a884b411602:36681 {}] regionserver.HRegionServer(1031): Exiting; stopping=2a884b411602,36681,1732226420789; zookeeper connection closed. 2024-11-21T22:00:30,690 DEBUG [pool-63-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36681-0x10160b31ce40001, quorum=127.0.0.1:62211, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T22:00:30,690 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1618469b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1618469b 2024-11-21T22:00:30,691 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-21T22:00:30,702 DEBUG [M:0;2a884b411602:38029 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/861e3b08-6197-a090-a4ff-052390850419/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88f2a104135547008ace29eab595ca0c is 2987, key is \x00\x00\x00\x00\x00\x00\x00