2024-11-22 12:13:14,586 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-22 12:13:14,598 main DEBUG Took 0.010086 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 12:13:14,598 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 12:13:14,599 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 12:13:14,600 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 12:13:14,601 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,607 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 12:13:14,619 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,621 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,622 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,623 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,624 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,624 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,625 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,626 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,627 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,628 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,628 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,629 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,630 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,630 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 12:13:14,630 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,631 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 12:13:14,632 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 12:13:14,633 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 12:13:14,635 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 12:13:14,635 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 12:13:14,637 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 12:13:14,637 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 12:13:14,645 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 12:13:14,647 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 12:13:14,649 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 12:13:14,650 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 12:13:14,650 main DEBUG createAppenders(={Console}) 2024-11-22 12:13:14,651 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-22 12:13:14,651 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-22 12:13:14,652 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-22 12:13:14,652 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 12:13:14,652 main DEBUG OutputStream closed 2024-11-22 12:13:14,653 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 12:13:14,653 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 12:13:14,653 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-22 12:13:14,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 12:13:14,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 12:13:14,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 12:13:14,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 12:13:14,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 12:13:14,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 12:13:14,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 12:13:14,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 12:13:14,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 12:13:14,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 12:13:14,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 12:13:14,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 12:13:14,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 12:13:14,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 12:13:14,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 12:13:14,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 12:13:14,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 12:13:14,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 12:13:14,737 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 12:13:14,737 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-22 12:13:14,737 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 12:13:14,738 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-22T12:13:14,751 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-22 12:13:14,753 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 12:13:14,753 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T12:13:15,083 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017 2024-11-22T12:13:15,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-22T12:13:15,142 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58, deleteOnExit=true 2024-11-22T12:13:15,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-22T12:13:15,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/test.cache.data in system properties and HBase conf 2024-11-22T12:13:15,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.tmp.dir in system properties and HBase conf 2024-11-22T12:13:15,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.log.dir in system properties and HBase conf 2024-11-22T12:13:15,149 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-22T12:13:15,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-22T12:13:15,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-22T12:13:15,254 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T12:13:15,389 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T12:13:15,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-22T12:13:15,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-22T12:13:15,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-22T12:13:15,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T12:13:15,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-22T12:13:15,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-22T12:13:15,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T12:13:15,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T12:13:15,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-22T12:13:15,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/nfs.dump.dir in system properties and HBase conf 2024-11-22T12:13:15,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/java.io.tmpdir in system properties and HBase conf 2024-11-22T12:13:15,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T12:13:15,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-22T12:13:15,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-22T12:13:16,720 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-22T12:13:16,855 INFO [Time-limited test {}] log.Log(170): Logging initialized @2990ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-22T12:13:16,962 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T12:13:17,036 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T12:13:17,081 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T12:13:17,081 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T12:13:17,085 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T12:13:17,112 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T12:13:17,117 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d9452bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.log.dir/,AVAILABLE} 2024-11-22T12:13:17,118 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d5b118c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T12:13:17,422 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3260f08c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/java.io.tmpdir/jetty-localhost-42769-hadoop-hdfs-3_4_1-tests_jar-_-any-15772819583488695500/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T12:13:17,436 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@369f03e8{HTTP/1.1, (http/1.1)}{localhost:42769} 2024-11-22T12:13:17,437 INFO [Time-limited test {}] server.Server(415): Started @3573ms 2024-11-22T12:13:18,253 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T12:13:18,260 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T12:13:18,261 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T12:13:18,261 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T12:13:18,261 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-22T12:13:18,262 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20ec9bac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.log.dir/,AVAILABLE} 2024-11-22T12:13:18,262 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5026a795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T12:13:18,361 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4f33137f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/java.io.tmpdir/jetty-localhost-46623-hadoop-hdfs-3_4_1-tests_jar-_-any-6299376510480503622/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T12:13:18,362 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6de9cba2{HTTP/1.1, (http/1.1)}{localhost:46623} 2024-11-22T12:13:18,362 INFO [Time-limited test {}] server.Server(415): Started @4498ms 2024-11-22T12:13:18,410 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T12:13:18,520 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T12:13:18,528 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T12:13:18,531 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T12:13:18,531 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T12:13:18,532 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T12:13:18,533 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b3caf0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.log.dir/,AVAILABLE} 2024-11-22T12:13:18,534 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@540e5e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T12:13:18,643 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7549e2f1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/java.io.tmpdir/jetty-localhost-34977-hadoop-hdfs-3_4_1-tests_jar-_-any-13677270212146498898/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T12:13:18,644 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53e0679{HTTP/1.1, (http/1.1)}{localhost:34977} 2024-11-22T12:13:18,644 INFO [Time-limited test {}] server.Server(415): Started @4780ms 2024-11-22T12:13:18,647 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T12:13:18,686 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T12:13:18,690 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T12:13:18,693 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T12:13:18,694 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T12:13:18,694 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T12:13:18,695 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@469f5a42{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.log.dir/,AVAILABLE} 2024-11-22T12:13:18,695 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75445ba5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T12:13:18,793 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6eecab6b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/java.io.tmpdir/jetty-localhost-39581-hadoop-hdfs-3_4_1-tests_jar-_-any-12036401277337068222/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T12:13:18,794 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3fb3fd4b{HTTP/1.1, (http/1.1)}{localhost:39581} 2024-11-22T12:13:18,794 INFO [Time-limited test {}] server.Server(415): Started @4930ms 2024-11-22T12:13:18,796 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T12:13:20,098 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data1/current/BP-1283605086-172.17.0.2-1732277596080/current, will proceed with Du for space computation calculation, 2024-11-22T12:13:20,098 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data4/current/BP-1283605086-172.17.0.2-1732277596080/current, will proceed with Du for space computation calculation, 2024-11-22T12:13:20,098 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data3/current/BP-1283605086-172.17.0.2-1732277596080/current, will proceed with Du for space computation calculation, 2024-11-22T12:13:20,098 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data2/current/BP-1283605086-172.17.0.2-1732277596080/current, will proceed with Du for space computation calculation, 2024-11-22T12:13:20,129 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T12:13:20,129 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T12:13:20,140 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data6/current/BP-1283605086-172.17.0.2-1732277596080/current, will proceed with Du for space computation calculation, 2024-11-22T12:13:20,140 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data5/current/BP-1283605086-172.17.0.2-1732277596080/current, will proceed with Du for space computation calculation, 2024-11-22T12:13:20,159 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T12:13:20,178 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaaf3e1d5365c392f with lease ID 0x90883ec84710bd66: Processing first storage report for DS-65ff8120-9702-4fea-8011-038991f8f8f0 from datanode DatanodeRegistration(127.0.0.1:33377, datanodeUuid=61144862-4f8d-45c6-bb17-cf3221e76363, infoPort=44637, infoSecurePort=0, ipcPort=36193, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080) 2024-11-22T12:13:20,179 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaaf3e1d5365c392f with lease ID 0x90883ec84710bd66: from storage DS-65ff8120-9702-4fea-8011-038991f8f8f0 node DatanodeRegistration(127.0.0.1:33377, datanodeUuid=61144862-4f8d-45c6-bb17-cf3221e76363, infoPort=44637, infoSecurePort=0, ipcPort=36193, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T12:13:20,179 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x72ebafa773402647 with lease ID 0x90883ec84710bd67: Processing first storage report for DS-0d6673b4-5e58-4255-9105-109b61968622 from datanode DatanodeRegistration(127.0.0.1:40437, datanodeUuid=4d200714-a1ac-4dbf-b045-a6021c003158, infoPort=37479, infoSecurePort=0, ipcPort=32773, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080) 2024-11-22T12:13:20,180 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x72ebafa773402647 with lease ID 0x90883ec84710bd67: from storage DS-0d6673b4-5e58-4255-9105-109b61968622 node DatanodeRegistration(127.0.0.1:40437, datanodeUuid=4d200714-a1ac-4dbf-b045-a6021c003158, infoPort=37479, infoSecurePort=0, ipcPort=32773, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-22T12:13:20,180 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f4a02c100cb1a5a with lease ID 0x90883ec84710bd65: Processing first storage report for DS-1b13ec14-000c-4db2-b97b-89138789e6a7 from datanode DatanodeRegistration(127.0.0.1:33883, datanodeUuid=513afaf4-2a3f-4084-895f-0011386d0f5c, infoPort=34481, infoSecurePort=0, ipcPort=40615, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080) 2024-11-22T12:13:20,180 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f4a02c100cb1a5a with lease ID 0x90883ec84710bd65: from storage DS-1b13ec14-000c-4db2-b97b-89138789e6a7 node DatanodeRegistration(127.0.0.1:33883, datanodeUuid=513afaf4-2a3f-4084-895f-0011386d0f5c, infoPort=34481, infoSecurePort=0, ipcPort=40615, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T12:13:20,180 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaaf3e1d5365c392f with lease ID 0x90883ec84710bd66: Processing first storage report for DS-85fe1d21-2044-4412-bf7c-d4fb68c4374e from datanode DatanodeRegistration(127.0.0.1:33377, datanodeUuid=61144862-4f8d-45c6-bb17-cf3221e76363, infoPort=44637, infoSecurePort=0, ipcPort=36193, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080) 2024-11-22T12:13:20,180 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaaf3e1d5365c392f with lease ID 0x90883ec84710bd66: from storage DS-85fe1d21-2044-4412-bf7c-d4fb68c4374e node DatanodeRegistration(127.0.0.1:33377, datanodeUuid=61144862-4f8d-45c6-bb17-cf3221e76363, infoPort=44637, infoSecurePort=0, ipcPort=36193, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-22T12:13:20,181 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x72ebafa773402647 with lease ID 0x90883ec84710bd67: Processing first storage report for DS-dc6ecfb9-6a8d-493e-8770-b2db4bd4a63c from datanode DatanodeRegistration(127.0.0.1:40437, datanodeUuid=4d200714-a1ac-4dbf-b045-a6021c003158, infoPort=37479, infoSecurePort=0, ipcPort=32773, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080) 2024-11-22T12:13:20,181 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x72ebafa773402647 with lease ID 0x90883ec84710bd67: from storage DS-dc6ecfb9-6a8d-493e-8770-b2db4bd4a63c node DatanodeRegistration(127.0.0.1:40437, datanodeUuid=4d200714-a1ac-4dbf-b045-a6021c003158, infoPort=37479, infoSecurePort=0, ipcPort=32773, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-22T12:13:20,181 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f4a02c100cb1a5a with lease ID 0x90883ec84710bd65: Processing first storage report for DS-ef817197-bf99-4398-98af-6f4258206561 from datanode DatanodeRegistration(127.0.0.1:33883, datanodeUuid=513afaf4-2a3f-4084-895f-0011386d0f5c, infoPort=34481, infoSecurePort=0, ipcPort=40615, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080) 2024-11-22T12:13:20,181 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f4a02c100cb1a5a with lease ID 0x90883ec84710bd65: from storage DS-ef817197-bf99-4398-98af-6f4258206561 node DatanodeRegistration(127.0.0.1:33883, datanodeUuid=513afaf4-2a3f-4084-895f-0011386d0f5c, infoPort=34481, infoSecurePort=0, ipcPort=40615, storageInfo=lv=-57;cid=testClusterID;nsid=913877444;c=1732277596080), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-22T12:13:20,219 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017 2024-11-22T12:13:20,284 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/zookeeper_0, clientPort=59102, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T12:13:20,294 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59102 2024-11-22T12:13:20,308 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:20,311 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:20,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741825_1001 (size=7) 2024-11-22T12:13:20,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741825_1001 (size=7) 2024-11-22T12:13:20,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741825_1001 (size=7) 2024-11-22T12:13:20,935 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c with version=8 2024-11-22T12:13:20,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/hbase-staging 2024-11-22T12:13:21,011 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T12:13:21,240 INFO [Time-limited test {}] client.ConnectionUtils(128): master/46e18b47f3d3:0 server-side Connection retries=45 2024-11-22T12:13:21,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:21,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:21,254 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T12:13:21,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:21,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T12:13:21,386 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T12:13:21,444 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T12:13:21,453 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T12:13:21,457 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T12:13:21,483 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2335 (auto-detected) 2024-11-22T12:13:21,484 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T12:13:21,502 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35109 2024-11-22T12:13:21,522 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35109 connecting to ZooKeeper ensemble=127.0.0.1:59102 2024-11-22T12:13:21,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:351090x0, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T12:13:21,651 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35109-0x10162767bb80000 connected 2024-11-22T12:13:21,750 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:21,753 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:21,763 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T12:13:21,767 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c, hbase.cluster.distributed=false 2024-11-22T12:13:21,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T12:13:21,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35109 2024-11-22T12:13:21,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35109 2024-11-22T12:13:21,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35109 2024-11-22T12:13:21,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35109 2024-11-22T12:13:21,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35109 2024-11-22T12:13:21,889 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/46e18b47f3d3:0 server-side Connection retries=45 2024-11-22T12:13:21,891 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:21,891 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:21,891 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T12:13:21,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:21,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T12:13:21,894 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T12:13:21,897 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T12:13:21,898 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38629 2024-11-22T12:13:21,900 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38629 connecting to ZooKeeper ensemble=127.0.0.1:59102 2024-11-22T12:13:21,901 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:21,905 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:21,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386290x0, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T12:13:21,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38629-0x10162767bb80001 connected 2024-11-22T12:13:21,918 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T12:13:21,922 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T12:13:21,929 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T12:13:21,932 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T12:13:21,937 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T12:13:21,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38629 2024-11-22T12:13:21,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38629 2024-11-22T12:13:21,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38629 2024-11-22T12:13:21,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38629 2024-11-22T12:13:21,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38629 2024-11-22T12:13:21,956 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/46e18b47f3d3:0 server-side Connection retries=45 2024-11-22T12:13:21,956 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:21,957 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:21,957 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T12:13:21,957 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:21,957 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T12:13:21,958 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T12:13:21,958 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T12:13:21,959 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40513 2024-11-22T12:13:21,961 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40513 connecting to ZooKeeper ensemble=127.0.0.1:59102 2024-11-22T12:13:21,961 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:21,966 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:21,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405130x0, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T12:13:21,982 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40513-0x10162767bb80002 connected 2024-11-22T12:13:21,982 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T12:13:21,983 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T12:13:21,984 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T12:13:21,985 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T12:13:21,988 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T12:13:21,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40513 2024-11-22T12:13:21,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40513 2024-11-22T12:13:21,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40513 2024-11-22T12:13:21,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40513 2024-11-22T12:13:21,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40513 2024-11-22T12:13:22,008 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/46e18b47f3d3:0 server-side Connection retries=45 2024-11-22T12:13:22,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:22,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:22,009 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T12:13:22,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T12:13:22,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T12:13:22,009 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T12:13:22,010 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T12:13:22,011 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37263 2024-11-22T12:13:22,012 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37263 connecting to ZooKeeper ensemble=127.0.0.1:59102 2024-11-22T12:13:22,013 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:22,015 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:22,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:372630x0, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T12:13:22,023 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37263-0x10162767bb80003 connected 2024-11-22T12:13:22,024 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T12:13:22,024 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T12:13:22,025 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T12:13:22,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T12:13:22,029 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T12:13:22,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37263 2024-11-22T12:13:22,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37263 2024-11-22T12:13:22,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37263 2024-11-22T12:13:22,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37263 2024-11-22T12:13:22,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37263 2024-11-22T12:13:22,049 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;46e18b47f3d3:35109 2024-11-22T12:13:22,050 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:22,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T12:13:22,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T12:13:22,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T12:13:22,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T12:13:22,067 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:22,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T12:13:22,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T12:13:22,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T12:13:22,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,098 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T12:13:22,099 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/46e18b47f3d3,35109,1732277601092 from backup master directory 2024-11-22T12:13:22,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:22,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T12:13:22,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T12:13:22,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T12:13:22,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T12:13:22,108 WARN [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T12:13:22,108 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:22,110 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T12:13:22,111 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T12:13:22,165 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/hbase.id] with ID: 5caf31ec-e0fb-4464-8e14-cb86b3459b61 2024-11-22T12:13:22,165 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/.tmp/hbase.id 2024-11-22T12:13:22,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741826_1002 (size=42) 2024-11-22T12:13:22,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741826_1002 (size=42) 2024-11-22T12:13:22,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741826_1002 (size=42) 2024-11-22T12:13:22,183 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/.tmp/hbase.id]:[hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/hbase.id] 2024-11-22T12:13:22,232 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T12:13:22,237 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T12:13:22,254 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-22T12:13:22,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741827_1003 (size=196) 2024-11-22T12:13:22,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741827_1003 (size=196) 2024-11-22T12:13:22,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741827_1003 (size=196) 2024-11-22T12:13:22,300 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T12:13:22,301 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T12:13:22,314 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T12:13:22,318 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T12:13:22,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741828_1004 (size=1189) 2024-11-22T12:13:22,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741828_1004 (size=1189) 2024-11-22T12:13:22,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741828_1004 (size=1189) 2024-11-22T12:13:22,366 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store 2024-11-22T12:13:22,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741829_1005 (size=34) 2024-11-22T12:13:22,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741829_1005 (size=34) 2024-11-22T12:13:22,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741829_1005 (size=34) 2024-11-22T12:13:22,392 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T12:13:22,396 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:22,398 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T12:13:22,398 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T12:13:22,399 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T12:13:22,401 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T12:13:22,401 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T12:13:22,401 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T12:13:22,402 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732277602398Disabling compacts and flushes for region at 1732277602398Disabling writes for close at 1732277602401 (+3 ms)Writing region close event to WAL at 1732277602401Closed at 1732277602401 2024-11-22T12:13:22,405 WARN [master/46e18b47f3d3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/.initializing 2024-11-22T12:13:22,405 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/WALs/46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:22,414 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T12:13:22,429 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=46e18b47f3d3%2C35109%2C1732277601092, suffix=, logDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/WALs/46e18b47f3d3,35109,1732277601092, archiveDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/oldWALs, maxLogs=10 2024-11-22T12:13:22,459 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/WALs/46e18b47f3d3,35109,1732277601092/46e18b47f3d3%2C35109%2C1732277601092.1732277602435, exclude list is [], retry=0 2024-11-22T12:13:22,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-0d6673b4-5e58-4255-9105-109b61968622,DISK] 2024-11-22T12:13:22,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33883,DS-1b13ec14-000c-4db2-b97b-89138789e6a7,DISK] 2024-11-22T12:13:22,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-65ff8120-9702-4fea-8011-038991f8f8f0,DISK] 2024-11-22T12:13:22,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-22T12:13:22,513 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/WALs/46e18b47f3d3,35109,1732277601092/46e18b47f3d3%2C35109%2C1732277601092.1732277602435 2024-11-22T12:13:22,514 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:44637:44637),(127.0.0.1/127.0.0.1:34481:34481)] 2024-11-22T12:13:22,514 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T12:13:22,515 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:22,517 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,518 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T12:13:22,580 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:22,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:22,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T12:13:22,586 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:22,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:22,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T12:13:22,591 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:22,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:22,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T12:13:22,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:22,596 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:22,597 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,600 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,601 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,606 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,607 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,610 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T12:13:22,614 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T12:13:22,618 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:22,619 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67142547, jitterRate=5.019158124923706E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T12:13:22,626 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732277602529Initializing all the Stores at 1732277602531 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277602531Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277602532 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277602532Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277602532Cleaning up temporary data from old regions at 1732277602607 (+75 ms)Region opened successfully at 1732277602626 (+19 ms) 2024-11-22T12:13:22,628 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T12:13:22,660 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a2a5b9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=46e18b47f3d3/172.17.0.2:0 2024-11-22T12:13:22,689 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T12:13:22,701 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:22,702 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:22,705 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:22,706 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T12:13:22,711 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T12:13:22,711 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T12:13:22,737 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T12:13:22,746 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T12:13:22,874 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T12:13:22,878 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T12:13:22,880 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T12:13:22,885 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T12:13:22,887 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T12:13:22,891 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T12:13:22,896 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T12:13:22,898 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T12:13:22,906 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T12:13:22,926 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T12:13:22,938 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T12:13:22,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T12:13:22,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T12:13:22,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T12:13:22,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T12:13:22,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,954 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=46e18b47f3d3,35109,1732277601092, sessionid=0x10162767bb80000, setting cluster-up flag (Was=false) 2024-11-22T12:13:22,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:22,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:23,012 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T12:13:23,017 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:23,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:23,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:23,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:23,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:23,223 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T12:13:23,225 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:23,231 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T12:13:23,236 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer(746): ClusterId : 5caf31ec-e0fb-4464-8e14-cb86b3459b61 2024-11-22T12:13:23,236 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(746): ClusterId : 5caf31ec-e0fb-4464-8e14-cb86b3459b61 2024-11-22T12:13:23,236 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(746): ClusterId : 5caf31ec-e0fb-4464-8e14-cb86b3459b61 2024-11-22T12:13:23,239 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T12:13:23,239 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T12:13:23,239 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T12:13:23,266 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T12:13:23,266 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T12:13:23,266 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T12:13:23,266 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T12:13:23,266 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T12:13:23,266 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T12:13:23,287 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T12:13:23,287 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T12:13:23,287 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T12:13:23,287 DEBUG [RS:2;46e18b47f3d3:37263 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d133ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=46e18b47f3d3/172.17.0.2:0 2024-11-22T12:13:23,288 DEBUG [RS:0;46e18b47f3d3:38629 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d101506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=46e18b47f3d3/172.17.0.2:0 2024-11-22T12:13:23,288 DEBUG [RS:1;46e18b47f3d3:40513 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97996b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=46e18b47f3d3/172.17.0.2:0 2024-11-22T12:13:23,301 DEBUG [RS:2;46e18b47f3d3:37263 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;46e18b47f3d3:37263 2024-11-22T12:13:23,305 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T12:13:23,305 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T12:13:23,305 DEBUG [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T12:13:23,306 DEBUG [RS:0;46e18b47f3d3:38629 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;46e18b47f3d3:38629 2024-11-22T12:13:23,306 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T12:13:23,306 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T12:13:23,306 DEBUG [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T12:13:23,307 DEBUG [RS:1;46e18b47f3d3:40513 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;46e18b47f3d3:40513 2024-11-22T12:13:23,306 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:23,307 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T12:13:23,307 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T12:13:23,307 DEBUG [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T12:13:23,308 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer(2659): reportForDuty to master=46e18b47f3d3,35109,1732277601092 with port=37263, startcode=1732277602008 2024-11-22T12:13:23,308 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(2659): reportForDuty to master=46e18b47f3d3,35109,1732277601092 with port=38629, startcode=1732277601859 2024-11-22T12:13:23,308 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(2659): reportForDuty to master=46e18b47f3d3,35109,1732277601092 with port=40513, startcode=1732277601956 2024-11-22T12:13:23,316 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T12:13:23,318 DEBUG [RS:0;46e18b47f3d3:38629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T12:13:23,318 DEBUG [RS:2;46e18b47f3d3:37263 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T12:13:23,318 DEBUG [RS:1;46e18b47f3d3:40513 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T12:13:23,323 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T12:13:23,328 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 46e18b47f3d3,35109,1732277601092 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T12:13:23,335 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/46e18b47f3d3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T12:13:23,335 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/46e18b47f3d3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T12:13:23,336 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/46e18b47f3d3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T12:13:23,336 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/46e18b47f3d3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T12:13:23,336 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/46e18b47f3d3:0, corePoolSize=10, maxPoolSize=10 2024-11-22T12:13:23,336 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,336 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/46e18b47f3d3:0, corePoolSize=2, maxPoolSize=2 2024-11-22T12:13:23,336 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,338 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277633338 2024-11-22T12:13:23,340 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T12:13:23,342 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T12:13:23,346 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T12:13:23,347 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T12:13:23,347 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T12:13:23,347 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T12:13:23,349 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:23,349 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T12:13:23,354 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57431, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T12:13:23,354 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36101, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T12:13:23,354 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42147, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T12:13:23,352 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,357 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:23,357 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T12:13:23,357 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T12:13:23,358 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T12:13:23,359 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T12:13:23,361 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T12:13:23,362 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T12:13:23,362 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:23,364 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/46e18b47f3d3:0:becomeActiveMaster-HFileCleaner.large.0-1732277603363,5,FailOnTimeoutGroup] 2024-11-22T12:13:23,365 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/46e18b47f3d3:0:becomeActiveMaster-HFileCleaner.small.0-1732277603365,5,FailOnTimeoutGroup] 2024-11-22T12:13:23,365 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,365 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T12:13:23,367 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,367 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,369 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:23,371 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] master.ServerManager(517): Registering regionserver=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:23,380 DEBUG [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:23,380 DEBUG [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36765 2024-11-22T12:13:23,380 DEBUG [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T12:13:23,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T12:13:23,391 DEBUG [RS:0;46e18b47f3d3:38629 {}] zookeeper.ZKUtil(111): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:23,391 WARN [RS:0;46e18b47f3d3:38629 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T12:13:23,392 INFO [RS:0;46e18b47f3d3:38629 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T12:13:23,392 DEBUG [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:23,394 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [46e18b47f3d3,38629,1732277601859] 2024-11-22T12:13:23,420 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:23,421 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] master.ServerManager(517): Registering regionserver=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:23,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741831_1007 (size=1321) 2024-11-22T12:13:23,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741831_1007 (size=1321) 2024-11-22T12:13:23,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741831_1007 (size=1321) 2024-11-22T12:13:23,428 DEBUG [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:23,428 DEBUG [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36765 2024-11-22T12:13:23,428 DEBUG [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T12:13:23,432 DEBUG [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-22T12:13:23,432 WARN [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-22T12:13:23,432 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T12:13:23,433 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:23,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741832_1008 (size=32) 2024-11-22T12:13:23,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741832_1008 (size=32) 2024-11-22T12:13:23,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741832_1008 (size=32) 2024-11-22T12:13:23,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:23,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T12:13:23,456 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T12:13:23,456 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:23,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:23,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T12:13:23,460 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T12:13:23,460 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:23,461 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:23,461 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T12:13:23,464 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T12:13:23,464 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:23,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:23,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T12:13:23,469 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T12:13:23,469 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:23,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:23,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T12:13:23,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T12:13:23,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740 2024-11-22T12:13:23,473 DEBUG [RS:2;46e18b47f3d3:37263 {}] zookeeper.ZKUtil(111): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:23,473 WARN [RS:2;46e18b47f3d3:37263 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T12:13:23,473 INFO [RS:2;46e18b47f3d3:37263 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T12:13:23,474 DEBUG [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:23,474 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740 2024-11-22T12:13:23,477 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T12:13:23,478 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T12:13:23,480 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T12:13:23,483 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T12:13:23,483 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [46e18b47f3d3,37263,1732277602008] 2024-11-22T12:13:23,487 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:23,488 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71855624, jitterRate=0.0707322359085083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T12:13:23,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732277603451Initializing all the Stores at 1732277603452 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277603452Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277603453 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277603453Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277603453Cleaning up temporary data from old regions at 1732277603478 (+25 ms)Region opened successfully at 1732277603493 (+15 ms) 2024-11-22T12:13:23,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T12:13:23,493 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T12:13:23,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T12:13:23,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T12:13:23,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T12:13:23,495 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T12:13:23,495 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732277603493Disabling compacts and flushes for region at 1732277603493Disabling writes for close at 1732277603493Writing region close event to WAL at 1732277603494 (+1 ms)Closed at 1732277603495 (+1 ms) 2024-11-22T12:13:23,495 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T12:13:23,495 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T12:13:23,498 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:23,498 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T12:13:23,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T12:13:23,515 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T12:13:23,516 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T12:13:23,517 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:23,521 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T12:13:23,521 INFO [RS:2;46e18b47f3d3:37263 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T12:13:23,521 INFO [RS:0;46e18b47f3d3:38629 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T12:13:23,521 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,521 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,522 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T12:13:23,522 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T12:13:23,528 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T12:13:23,528 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T12:13:23,529 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,529 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,530 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0, corePoolSize=2, maxPoolSize=2 2024-11-22T12:13:23,530 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,530 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0, corePoolSize=2, maxPoolSize=2 2024-11-22T12:13:23,531 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,531 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/46e18b47f3d3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T12:13:23,531 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,532 DEBUG [RS:2;46e18b47f3d3:37263 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/46e18b47f3d3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T12:13:23,532 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,532 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/46e18b47f3d3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T12:13:23,532 DEBUG [RS:0;46e18b47f3d3:38629 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/46e18b47f3d3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T12:13:23,533 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(2659): reportForDuty to master=46e18b47f3d3,35109,1732277601092 with port=40513, startcode=1732277601956 2024-11-22T12:13:23,534 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,534 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:23,535 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,37263,1732277602008-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] master.ServerManager(517): Registering regionserver=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:23,535 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,535 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,38629,1732277601859-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T12:13:23,538 DEBUG [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:23,538 DEBUG [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36765 2024-11-22T12:13:23,538 DEBUG [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T12:13:23,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T12:13:23,560 DEBUG [RS:1;46e18b47f3d3:40513 {}] zookeeper.ZKUtil(111): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:23,560 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T12:13:23,560 WARN [RS:1;46e18b47f3d3:40513 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T12:13:23,560 INFO [RS:1;46e18b47f3d3:40513 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T12:13:23,560 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [46e18b47f3d3,40513,1732277601956] 2024-11-22T12:13:23,560 DEBUG [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:23,562 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,37263,1732277602008-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,563 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,563 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.Replication(171): 46e18b47f3d3,37263,1732277602008 started 2024-11-22T12:13:23,567 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T12:13:23,567 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,38629,1732277601859-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,567 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,567 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.Replication(171): 46e18b47f3d3,38629,1732277601859 started 2024-11-22T12:13:23,569 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T12:13:23,572 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T12:13:23,578 INFO [RS:1;46e18b47f3d3:40513 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T12:13:23,579 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,579 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T12:13:23,581 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T12:13:23,581 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,582 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,582 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,582 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,582 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,582 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,583 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0, corePoolSize=2, maxPoolSize=2 2024-11-22T12:13:23,583 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,583 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,583 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,583 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,583 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,583 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/46e18b47f3d3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T12:13:23,584 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/46e18b47f3d3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T12:13:23,584 DEBUG [RS:1;46e18b47f3d3:40513 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/46e18b47f3d3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T12:13:23,588 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,588 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,589 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,589 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,589 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,589 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer(1482): Serving as 46e18b47f3d3,37263,1732277602008, RpcServer on 46e18b47f3d3/172.17.0.2:37263, sessionid=0x10162767bb80003 2024-11-22T12:13:23,589 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,589 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,40513,1732277601956-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T12:13:23,590 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T12:13:23,590 DEBUG [RS:2;46e18b47f3d3:37263 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:23,590 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,590 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(1482): Serving as 46e18b47f3d3,38629,1732277601859, RpcServer on 46e18b47f3d3/172.17.0.2:38629, sessionid=0x10162767bb80001 2024-11-22T12:13:23,590 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '46e18b47f3d3,37263,1732277602008' 2024-11-22T12:13:23,590 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T12:13:23,590 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T12:13:23,591 DEBUG [RS:0;46e18b47f3d3:38629 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:23,591 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '46e18b47f3d3,38629,1732277601859' 2024-11-22T12:13:23,591 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T12:13:23,592 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T12:13:23,592 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T12:13:23,592 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T12:13:23,592 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T12:13:23,592 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T12:13:23,592 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T12:13:23,592 DEBUG [RS:2;46e18b47f3d3:37263 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:23,592 DEBUG [RS:0;46e18b47f3d3:38629 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:23,593 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '46e18b47f3d3,37263,1732277602008' 2024-11-22T12:13:23,593 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '46e18b47f3d3,38629,1732277601859' 2024-11-22T12:13:23,593 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T12:13:23,593 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T12:13:23,593 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T12:13:23,594 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T12:13:23,594 DEBUG [RS:2;46e18b47f3d3:37263 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T12:13:23,594 DEBUG [RS:0;46e18b47f3d3:38629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T12:13:23,594 INFO [RS:0;46e18b47f3d3:38629 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T12:13:23,594 INFO [RS:2;46e18b47f3d3:37263 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T12:13:23,595 INFO [RS:0;46e18b47f3d3:38629 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T12:13:23,595 INFO [RS:2;46e18b47f3d3:37263 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T12:13:23,610 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T12:13:23,610 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,40513,1732277601956-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,610 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,611 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.Replication(171): 46e18b47f3d3,40513,1732277601956 started 2024-11-22T12:13:23,626 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:23,627 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(1482): Serving as 46e18b47f3d3,40513,1732277601956, RpcServer on 46e18b47f3d3/172.17.0.2:40513, sessionid=0x10162767bb80002 2024-11-22T12:13:23,627 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T12:13:23,627 DEBUG [RS:1;46e18b47f3d3:40513 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:23,627 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '46e18b47f3d3,40513,1732277601956' 2024-11-22T12:13:23,627 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T12:13:23,628 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T12:13:23,628 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T12:13:23,628 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T12:13:23,629 DEBUG [RS:1;46e18b47f3d3:40513 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:23,629 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '46e18b47f3d3,40513,1732277601956' 2024-11-22T12:13:23,629 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T12:13:23,629 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T12:13:23,630 DEBUG [RS:1;46e18b47f3d3:40513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T12:13:23,630 INFO [RS:1;46e18b47f3d3:40513 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T12:13:23,630 INFO [RS:1;46e18b47f3d3:40513 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T12:13:23,671 WARN [46e18b47f3d3:35109 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T12:13:23,699 INFO [RS:2;46e18b47f3d3:37263 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T12:13:23,699 INFO [RS:0;46e18b47f3d3:38629 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T12:13:23,702 INFO [RS:2;46e18b47f3d3:37263 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=46e18b47f3d3%2C37263%2C1732277602008, suffix=, logDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008, archiveDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs, maxLogs=32 2024-11-22T12:13:23,702 INFO [RS:0;46e18b47f3d3:38629 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=46e18b47f3d3%2C38629%2C1732277601859, suffix=, logDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,38629,1732277601859, archiveDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs, maxLogs=32 2024-11-22T12:13:23,719 DEBUG [RS:2;46e18b47f3d3:37263 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008/46e18b47f3d3%2C37263%2C1732277602008.1732277603705, exclude list is [], retry=0 2024-11-22T12:13:23,723 DEBUG [RS:0;46e18b47f3d3:38629 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,38629,1732277601859/46e18b47f3d3%2C38629%2C1732277601859.1732277603706, exclude list is [], retry=0 2024-11-22T12:13:23,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-0d6673b4-5e58-4255-9105-109b61968622,DISK] 2024-11-22T12:13:23,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-65ff8120-9702-4fea-8011-038991f8f8f0,DISK] 2024-11-22T12:13:23,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33883,DS-1b13ec14-000c-4db2-b97b-89138789e6a7,DISK] 2024-11-22T12:13:23,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-65ff8120-9702-4fea-8011-038991f8f8f0,DISK] 2024-11-22T12:13:23,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-0d6673b4-5e58-4255-9105-109b61968622,DISK] 2024-11-22T12:13:23,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33883,DS-1b13ec14-000c-4db2-b97b-89138789e6a7,DISK] 2024-11-22T12:13:23,730 INFO [RS:2;46e18b47f3d3:37263 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008/46e18b47f3d3%2C37263%2C1732277602008.1732277603705 2024-11-22T12:13:23,731 INFO [RS:1;46e18b47f3d3:40513 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T12:13:23,733 INFO [RS:1;46e18b47f3d3:40513 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=46e18b47f3d3%2C40513%2C1732277601956, suffix=, logDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,40513,1732277601956, archiveDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs, maxLogs=32 2024-11-22T12:13:23,740 DEBUG [RS:2;46e18b47f3d3:37263 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44637:44637),(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:34481:34481)] 2024-11-22T12:13:23,742 INFO [RS:0;46e18b47f3d3:38629 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,38629,1732277601859/46e18b47f3d3%2C38629%2C1732277601859.1732277603706 2024-11-22T12:13:23,743 DEBUG [RS:0;46e18b47f3d3:38629 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44637:44637),(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:34481:34481)] 2024-11-22T12:13:23,751 DEBUG [RS:1;46e18b47f3d3:40513 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,40513,1732277601956/46e18b47f3d3%2C40513%2C1732277601956.1732277603735, exclude list is [], retry=0 2024-11-22T12:13:23,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-65ff8120-9702-4fea-8011-038991f8f8f0,DISK] 2024-11-22T12:13:23,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33883,DS-1b13ec14-000c-4db2-b97b-89138789e6a7,DISK] 2024-11-22T12:13:23,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-0d6673b4-5e58-4255-9105-109b61968622,DISK] 2024-11-22T12:13:23,768 INFO [RS:1;46e18b47f3d3:40513 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,40513,1732277601956/46e18b47f3d3%2C40513%2C1732277601956.1732277603735 2024-11-22T12:13:23,769 DEBUG [RS:1;46e18b47f3d3:40513 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34481:34481),(127.0.0.1/127.0.0.1:44637:44637),(127.0.0.1/127.0.0.1:37479:37479)] 2024-11-22T12:13:23,924 DEBUG [46e18b47f3d3:35109 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-22T12:13:23,935 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(204): Hosts are {46e18b47f3d3=0} racks are {/default-rack=0} 2024-11-22T12:13:23,941 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T12:13:23,941 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T12:13:23,941 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-22T12:13:23,941 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T12:13:23,941 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T12:13:23,941 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-22T12:13:23,941 INFO [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T12:13:23,941 INFO [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T12:13:23,941 INFO [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-22T12:13:23,941 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T12:13:23,947 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:23,953 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,37263,1732277602008, state=OPENING 2024-11-22T12:13:23,980 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T12:13:24,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:24,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:24,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:24,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:24,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:24,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:24,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:24,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:24,071 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:24,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,37263,1732277602008}] 2024-11-22T12:13:24,257 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T12:13:24,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60471, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T12:13:24,270 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T12:13:24,270 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T12:13:24,271 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T12:13:24,274 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=46e18b47f3d3%2C37263%2C1732277602008.meta, suffix=.meta, logDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008, archiveDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs, maxLogs=32 2024-11-22T12:13:24,292 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, exclude list is [], retry=0 2024-11-22T12:13:24,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-0d6673b4-5e58-4255-9105-109b61968622,DISK] 2024-11-22T12:13:24,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33883,DS-1b13ec14-000c-4db2-b97b-89138789e6a7,DISK] 2024-11-22T12:13:24,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-65ff8120-9702-4fea-8011-038991f8f8f0,DISK] 2024-11-22T12:13:24,299 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta 2024-11-22T12:13:24,300 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:34481:34481),(127.0.0.1/127.0.0.1:44637:44637)] 2024-11-22T12:13:24,300 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T12:13:24,302 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T12:13:24,305 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T12:13:24,311 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T12:13:24,315 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T12:13:24,316 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,316 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T12:13:24,316 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T12:13:24,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T12:13:24,321 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T12:13:24,321 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:24,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:24,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T12:13:24,324 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T12:13:24,324 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:24,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:24,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T12:13:24,327 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T12:13:24,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:24,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:24,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T12:13:24,329 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T12:13:24,329 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:24,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:24,331 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T12:13:24,332 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740 2024-11-22T12:13:24,335 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740 2024-11-22T12:13:24,337 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T12:13:24,337 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T12:13:24,338 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T12:13:24,341 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T12:13:24,342 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66338500, jitterRate=-0.011479318141937256}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T12:13:24,342 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T12:13:24,343 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732277604317Writing region info on filesystem at 1732277604317Initializing all the Stores at 1732277604319 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277604319Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277604319Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277604319Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277604320 (+1 ms)Cleaning up temporary data from old regions at 1732277604337 (+17 ms)Running coprocessor post-open hooks at 1732277604342 (+5 ms)Region opened successfully at 1732277604343 (+1 ms) 2024-11-22T12:13:24,350 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732277604247 2024-11-22T12:13:24,361 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T12:13:24,362 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T12:13:24,364 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:24,366 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,37263,1732277602008, state=OPEN 2024-11-22T12:13:24,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:24,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:24,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:24,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:24,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:24,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:24,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:24,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:24,434 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:24,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T12:13:24,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,37263,1732277602008 in 360 msec 2024-11-22T12:13:24,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T12:13:24,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 939 msec 2024-11-22T12:13:24,453 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:24,453 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T12:13:24,471 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T12:13:24,473 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1] 2024-11-22T12:13:24,492 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T12:13:24,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44323, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T12:13:24,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2550 sec 2024-11-22T12:13:24,518 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732277604518, completionTime=-1 2024-11-22T12:13:24,521 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-22T12:13:24,522 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:24,544 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-22T12:13:24,544 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277664544 2024-11-22T12:13:24,544 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277724544 2024-11-22T12:13:24,544 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-11-22T12:13:24,547 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:24,556 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,35109,1732277601092-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:24,556 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,35109,1732277601092-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:24,556 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,35109,1732277601092-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:24,558 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-46e18b47f3d3:35109, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:24,558 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:24,559 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:24,564 DEBUG [master/46e18b47f3d3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T12:13:24,588 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.480sec 2024-11-22T12:13:24,590 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T12:13:24,591 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T12:13:24,593 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T12:13:24,593 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T12:13:24,593 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T12:13:24,594 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,35109,1732277601092-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T12:13:24,595 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,35109,1732277601092-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T12:13:24,600 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T12:13:24,601 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T12:13:24,602 INFO [master/46e18b47f3d3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46e18b47f3d3,35109,1732277601092-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T12:13:24,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3475a1ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T12:13:24,653 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T12:13:24,653 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T12:13:24,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 46e18b47f3d3,35109,-1 for getting cluster id 2024-11-22T12:13:24,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T12:13:24,668 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5caf31ec-e0fb-4464-8e14-cb86b3459b61' 2024-11-22T12:13:24,671 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T12:13:24,671 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5caf31ec-e0fb-4464-8e14-cb86b3459b61" 2024-11-22T12:13:24,671 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df07cfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T12:13:24,672 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [46e18b47f3d3,35109,-1] 2024-11-22T12:13:24,674 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T12:13:24,677 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:24,677 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T12:13:24,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e978e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T12:13:24,681 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T12:13:24,688 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1] 2024-11-22T12:13:24,689 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T12:13:24,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T12:13:24,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:24,738 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T12:13:24,743 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:24,745 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@284ccd7e 2024-11-22T12:13:24,746 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T12:13:24,752 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T12:13:24,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T12:13:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:24,771 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T12:13:24,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-22T12:13:24,774 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:24,778 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T12:13:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T12:13:24,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741837_1013 (size=358) 2024-11-22T12:13:24,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741837_1013 (size=358) 2024-11-22T12:13:24,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741837_1013 (size=358) 2024-11-22T12:13:24,796 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ff71fd9f56108f32cc070df1485ba23, NAME => 'test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,799 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7cf86cd087fe778d3cb9ffcdb4f32fdf, NAME => 'test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,799 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 142956bd7cdd41ce32216563195a828d, NAME => 'test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741838_1014 (size=42) 2024-11-22T12:13:24,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741838_1014 (size=42) 2024-11-22T12:13:24,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741838_1014 (size=42) 2024-11-22T12:13:24,820 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,820 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6ff71fd9f56108f32cc070df1485ba23, disabling compactions & flushes 2024-11-22T12:13:24,820 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:24,820 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:24,820 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. after waiting 0 ms 2024-11-22T12:13:24,820 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:24,820 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:24,820 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ff71fd9f56108f32cc070df1485ba23: Waiting for close lock at 1732277604820Disabling compacts and flushes for region at 1732277604820Disabling writes for close at 1732277604820Writing region close event to WAL at 1732277604820Closed at 1732277604820 2024-11-22T12:13:24,821 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 800c34543b39c820cefb84a25e72255c, NAME => 'test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741839_1015 (size=45) 2024-11-22T12:13:24,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741839_1015 (size=45) 2024-11-22T12:13:24,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741839_1015 (size=45) 2024-11-22T12:13:24,832 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,833 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 142956bd7cdd41ce32216563195a828d, disabling compactions & flushes 2024-11-22T12:13:24,833 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:24,833 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:24,833 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. after waiting 0 ms 2024-11-22T12:13:24,833 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:24,833 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:24,833 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 142956bd7cdd41ce32216563195a828d: Waiting for close lock at 1732277604833Disabling compacts and flushes for region at 1732277604833Disabling writes for close at 1732277604833Writing region close event to WAL at 1732277604833Closed at 1732277604833 2024-11-22T12:13:24,833 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7aef6551d501af493aaa4fd1077bf9c9, NAME => 'test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741840_1016 (size=45) 2024-11-22T12:13:24,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741840_1016 (size=45) 2024-11-22T12:13:24,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741840_1016 (size=45) 2024-11-22T12:13:24,841 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,841 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 7cf86cd087fe778d3cb9ffcdb4f32fdf, disabling compactions & flushes 2024-11-22T12:13:24,841 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:24,841 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:24,841 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. after waiting 0 ms 2024-11-22T12:13:24,841 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:24,841 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:24,841 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7cf86cd087fe778d3cb9ffcdb4f32fdf: Waiting for close lock at 1732277604841Disabling compacts and flushes for region at 1732277604841Disabling writes for close at 1732277604841Writing region close event to WAL at 1732277604841Closed at 1732277604841 2024-11-22T12:13:24,842 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d1e0cdf60c0d514100f271449411b4ed, NAME => 'test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741842_1018 (size=45) 2024-11-22T12:13:24,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741842_1018 (size=45) 2024-11-22T12:13:24,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741842_1018 (size=45) 2024-11-22T12:13:24,852 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,852 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7aef6551d501af493aaa4fd1077bf9c9, disabling compactions & flushes 2024-11-22T12:13:24,852 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:24,852 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:24,852 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. after waiting 0 ms 2024-11-22T12:13:24,852 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:24,852 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:24,852 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7aef6551d501af493aaa4fd1077bf9c9: Waiting for close lock at 1732277604852Disabling compacts and flushes for region at 1732277604852Disabling writes for close at 1732277604852Writing region close event to WAL at 1732277604852Closed at 1732277604852 2024-11-22T12:13:24,853 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b4584ab1e18d121599a608cdb4be19ea, NAME => 'test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741841_1017 (size=45) 2024-11-22T12:13:24,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741841_1017 (size=45) 2024-11-22T12:13:24,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741841_1017 (size=45) 2024-11-22T12:13:24,857 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,857 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 800c34543b39c820cefb84a25e72255c, disabling compactions & flushes 2024-11-22T12:13:24,857 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:24,857 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:24,857 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. after waiting 0 ms 2024-11-22T12:13:24,857 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:24,857 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:24,857 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 800c34543b39c820cefb84a25e72255c: Waiting for close lock at 1732277604857Disabling compacts and flushes for region at 1732277604857Disabling writes for close at 1732277604857Writing region close event to WAL at 1732277604857Closed at 1732277604857 2024-11-22T12:13:24,858 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e33fb94d3dd5df2c85edbc81c792b792, NAME => 'test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741843_1019 (size=45) 2024-11-22T12:13:24,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741843_1019 (size=45) 2024-11-22T12:13:24,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741843_1019 (size=45) 2024-11-22T12:13:24,869 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,869 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d1e0cdf60c0d514100f271449411b4ed, disabling compactions & flushes 2024-11-22T12:13:24,869 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:24,869 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:24,869 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. after waiting 0 ms 2024-11-22T12:13:24,869 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:24,869 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:24,869 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d1e0cdf60c0d514100f271449411b4ed: Waiting for close lock at 1732277604869Disabling compacts and flushes for region at 1732277604869Disabling writes for close at 1732277604869Writing region close event to WAL at 1732277604869Closed at 1732277604869 2024-11-22T12:13:24,870 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d5254a456e6265d346df2dc9fbb1afe4, NAME => 'test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741844_1020 (size=45) 2024-11-22T12:13:24,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741844_1020 (size=45) 2024-11-22T12:13:24,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741844_1020 (size=45) 2024-11-22T12:13:24,881 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,881 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b4584ab1e18d121599a608cdb4be19ea, disabling compactions & flushes 2024-11-22T12:13:24,881 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:24,881 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:24,881 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. after waiting 0 ms 2024-11-22T12:13:24,881 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:24,882 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:24,882 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b4584ab1e18d121599a608cdb4be19ea: Waiting for close lock at 1732277604881Disabling compacts and flushes for region at 1732277604881Disabling writes for close at 1732277604881Writing region close event to WAL at 1732277604881Closed at 1732277604881 2024-11-22T12:13:24,882 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b05283d6644e96595068a6390bc0e1fe, NAME => 'test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T12:13:24,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741845_1021 (size=45) 2024-11-22T12:13:24,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741845_1021 (size=45) 2024-11-22T12:13:24,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741845_1021 (size=45) 2024-11-22T12:13:24,889 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,889 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e33fb94d3dd5df2c85edbc81c792b792, disabling compactions & flushes 2024-11-22T12:13:24,889 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:24,889 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:24,889 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. after waiting 0 ms 2024-11-22T12:13:24,889 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:24,889 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:24,889 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e33fb94d3dd5df2c85edbc81c792b792: Waiting for close lock at 1732277604889Disabling compacts and flushes for region at 1732277604889Disabling writes for close at 1732277604889Writing region close event to WAL at 1732277604889Closed at 1732277604889 2024-11-22T12:13:24,890 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c48682b2ea1380365b4d638dd13d2a9, NAME => 'test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741846_1022 (size=45) 2024-11-22T12:13:24,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741846_1022 (size=45) 2024-11-22T12:13:24,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741846_1022 (size=45) 2024-11-22T12:13:24,909 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,909 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d5254a456e6265d346df2dc9fbb1afe4, disabling compactions & flushes 2024-11-22T12:13:24,909 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. 2024-11-22T12:13:24,909 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. 2024-11-22T12:13:24,909 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. after waiting 0 ms 2024-11-22T12:13:24,909 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. 2024-11-22T12:13:24,909 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. 2024-11-22T12:13:24,910 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d5254a456e6265d346df2dc9fbb1afe4: Waiting for close lock at 1732277604909Disabling compacts and flushes for region at 1732277604909Disabling writes for close at 1732277604909Writing region close event to WAL at 1732277604909Closed at 1732277604909 2024-11-22T12:13:24,910 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c3b4530e36a471e0bfd4c64c45ec26d5, NAME => 'test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741847_1023 (size=45) 2024-11-22T12:13:24,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741848_1024 (size=45) 2024-11-22T12:13:24,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741847_1023 (size=45) 2024-11-22T12:13:24,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741848_1024 (size=45) 2024-11-22T12:13:24,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741847_1023 (size=45) 2024-11-22T12:13:24,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741848_1024 (size=45) 2024-11-22T12:13:24,915 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,915 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b05283d6644e96595068a6390bc0e1fe, disabling compactions & flushes 2024-11-22T12:13:24,916 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. after waiting 0 ms 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6c48682b2ea1380365b4d638dd13d2a9, disabling compactions & flushes 2024-11-22T12:13:24,916 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:24,916 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b05283d6644e96595068a6390bc0e1fe: Waiting for close lock at 1732277604915Disabling compacts and flushes for region at 1732277604915Disabling writes for close at 1732277604916 (+1 ms)Writing region close event to WAL at 1732277604916Closed at 1732277604916 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. after waiting 0 ms 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:24,916 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:24,916 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c48682b2ea1380365b4d638dd13d2a9: Waiting for close lock at 1732277604916Disabling compacts and flushes for region at 1732277604916Disabling writes for close at 1732277604916Writing region close event to WAL at 1732277604916Closed at 1732277604916 2024-11-22T12:13:24,917 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => eb5b2e2a4461c2824dc11a5a423a6d19, NAME => 'test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,917 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 794bd231e579bf5991f118c3359c87ab, NAME => 'test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741849_1025 (size=45) 2024-11-22T12:13:24,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741849_1025 (size=45) 2024-11-22T12:13:24,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741849_1025 (size=45) 2024-11-22T12:13:24,933 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,933 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c3b4530e36a471e0bfd4c64c45ec26d5, disabling compactions & flushes 2024-11-22T12:13:24,933 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. 2024-11-22T12:13:24,933 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. 2024-11-22T12:13:24,933 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. after waiting 0 ms 2024-11-22T12:13:24,933 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. 2024-11-22T12:13:24,933 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. 2024-11-22T12:13:24,933 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c3b4530e36a471e0bfd4c64c45ec26d5: Waiting for close lock at 1732277604933Disabling compacts and flushes for region at 1732277604933Disabling writes for close at 1732277604933Writing region close event to WAL at 1732277604933Closed at 1732277604933 2024-11-22T12:13:24,934 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1c19c84c4949f0cb43eed608a20910dc, NAME => 'test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741850_1026 (size=45) 2024-11-22T12:13:24,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741850_1026 (size=45) 2024-11-22T12:13:24,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741850_1026 (size=45) 2024-11-22T12:13:24,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing eb5b2e2a4461c2824dc11a5a423a6d19, disabling compactions & flushes 2024-11-22T12:13:24,938 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:24,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:24,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. after waiting 0 ms 2024-11-22T12:13:24,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:24,938 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:24,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for eb5b2e2a4461c2824dc11a5a423a6d19: Waiting for close lock at 1732277604938Disabling compacts and flushes for region at 1732277604938Disabling writes for close at 1732277604938Writing region close event to WAL at 1732277604938Closed at 1732277604938 2024-11-22T12:13:24,939 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5acff54db0b679ccee30c4cfa789fe5b, NAME => 'test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741851_1027 (size=45) 2024-11-22T12:13:24,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741851_1027 (size=45) 2024-11-22T12:13:24,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741851_1027 (size=45) 2024-11-22T12:13:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741852_1028 (size=45) 2024-11-22T12:13:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741852_1028 (size=45) 2024-11-22T12:13:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741852_1028 (size=45) 2024-11-22T12:13:24,957 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,958 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1c19c84c4949f0cb43eed608a20910dc, disabling compactions & flushes 2024-11-22T12:13:24,958 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:24,958 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:24,958 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. after waiting 0 ms 2024-11-22T12:13:24,958 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:24,958 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:24,958 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1c19c84c4949f0cb43eed608a20910dc: Waiting for close lock at 1732277604958Disabling compacts and flushes for region at 1732277604958Disabling writes for close at 1732277604958Writing region close event to WAL at 1732277604958Closed at 1732277604958 2024-11-22T12:13:24,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741853_1029 (size=45) 2024-11-22T12:13:24,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741853_1029 (size=45) 2024-11-22T12:13:24,959 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 88a8bf4fbe7648785154bd80df699f0b, NAME => 'test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741853_1029 (size=45) 2024-11-22T12:13:24,961 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,961 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5acff54db0b679ccee30c4cfa789fe5b, disabling compactions & flushes 2024-11-22T12:13:24,961 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:24,961 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:24,961 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. after waiting 0 ms 2024-11-22T12:13:24,961 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:24,961 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:24,961 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5acff54db0b679ccee30c4cfa789fe5b: Waiting for close lock at 1732277604961Disabling compacts and flushes for region at 1732277604961Disabling writes for close at 1732277604961Writing region close event to WAL at 1732277604961Closed at 1732277604961 2024-11-22T12:13:24,962 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6dbb25f70e3828779de1b8447a41064c, NAME => 'test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741854_1030 (size=45) 2024-11-22T12:13:24,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741854_1030 (size=45) 2024-11-22T12:13:24,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741854_1030 (size=45) 2024-11-22T12:13:24,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741855_1031 (size=45) 2024-11-22T12:13:24,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741855_1031 (size=45) 2024-11-22T12:13:24,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741855_1031 (size=45) 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6dbb25f70e3828779de1b8447a41064c, disabling compactions & flushes 2024-11-22T12:13:24,980 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 88a8bf4fbe7648785154bd80df699f0b, disabling compactions & flushes 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:24,980 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. after waiting 0 ms 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:24,980 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. after waiting 0 ms 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6dbb25f70e3828779de1b8447a41064c: Waiting for close lock at 1732277604980Disabling compacts and flushes for region at 1732277604980Disabling writes for close at 1732277604980Writing region close event to WAL at 1732277604980Closed at 1732277604980 2024-11-22T12:13:24,980 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:24,980 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:24,981 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 88a8bf4fbe7648785154bd80df699f0b: Waiting for close lock at 1732277604980Disabling compacts and flushes for region at 1732277604980Disabling writes for close at 1732277604980Writing region close event to WAL at 1732277604980Closed at 1732277604980 2024-11-22T12:13:24,981 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4c61ef28412030a5de1748e0494b5d77, NAME => 'test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,981 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cca3907a8067af4c43fffaebaa942770, NAME => 'test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741856_1032 (size=45) 2024-11-22T12:13:24,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741857_1033 (size=45) 2024-11-22T12:13:24,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741857_1033 (size=45) 2024-11-22T12:13:24,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741856_1032 (size=45) 2024-11-22T12:13:24,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741856_1032 (size=45) 2024-11-22T12:13:24,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741857_1033 (size=45) 2024-11-22T12:13:24,994 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,994 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4c61ef28412030a5de1748e0494b5d77, disabling compactions & flushes 2024-11-22T12:13:24,994 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:24,994 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:24,994 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:24,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. after waiting 0 ms 2024-11-22T12:13:24,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:24,995 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cca3907a8067af4c43fffaebaa942770, disabling compactions & flushes 2024-11-22T12:13:24,995 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:24,995 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:24,995 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:24,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4c61ef28412030a5de1748e0494b5d77: Waiting for close lock at 1732277604994Disabling compacts and flushes for region at 1732277604994Disabling writes for close at 1732277604995 (+1 ms)Writing region close event to WAL at 1732277604995Closed at 1732277604995 2024-11-22T12:13:24,995 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. after waiting 0 ms 2024-11-22T12:13:24,995 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:24,995 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:24,995 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cca3907a8067af4c43fffaebaa942770: Waiting for close lock at 1732277604995Disabling compacts and flushes for region at 1732277604995Disabling writes for close at 1732277604995Writing region close event to WAL at 1732277604995Closed at 1732277604995 2024-11-22T12:13:24,995 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7f061dd4aec8e910fcda8dcf65d18b2b, NAME => 'test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:24,996 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 88ee6f91206e330acd4027ac1712c4b5, NAME => 'test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:25,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741859_1035 (size=45) 2024-11-22T12:13:25,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741859_1035 (size=45) 2024-11-22T12:13:25,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741859_1035 (size=45) 2024-11-22T12:13:25,008 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,008 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7f061dd4aec8e910fcda8dcf65d18b2b, disabling compactions & flushes 2024-11-22T12:13:25,008 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:25,008 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:25,008 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. after waiting 0 ms 2024-11-22T12:13:25,008 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:25,008 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:25,008 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7f061dd4aec8e910fcda8dcf65d18b2b: Waiting for close lock at 1732277605008Disabling compacts and flushes for region at 1732277605008Disabling writes for close at 1732277605008Writing region close event to WAL at 1732277605008Closed at 1732277605008 2024-11-22T12:13:25,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741858_1034 (size=45) 2024-11-22T12:13:25,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741858_1034 (size=45) 2024-11-22T12:13:25,009 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d1716ab8b0523ebd9a28d11dac1fd8e6, NAME => 'test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:25,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741858_1034 (size=45) 2024-11-22T12:13:25,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 88ee6f91206e330acd4027ac1712c4b5, disabling compactions & flushes 2024-11-22T12:13:25,010 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:25,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:25,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. after waiting 0 ms 2024-11-22T12:13:25,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:25,010 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:25,011 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 88ee6f91206e330acd4027ac1712c4b5: Waiting for close lock at 1732277605010Disabling compacts and flushes for region at 1732277605010Disabling writes for close at 1732277605010Writing region close event to WAL at 1732277605010Closed at 1732277605010 2024-11-22T12:13:25,011 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b745bd27cbcb9cc89fc04815d334ae24, NAME => 'test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:25,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741860_1036 (size=45) 2024-11-22T12:13:25,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741860_1036 (size=45) 2024-11-22T12:13:25,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741860_1036 (size=45) 2024-11-22T12:13:25,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d1716ab8b0523ebd9a28d11dac1fd8e6, disabling compactions & flushes 2024-11-22T12:13:25,021 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:25,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:25,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. after waiting 0 ms 2024-11-22T12:13:25,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:25,021 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:25,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d1716ab8b0523ebd9a28d11dac1fd8e6: Waiting for close lock at 1732277605021Disabling compacts and flushes for region at 1732277605021Disabling writes for close at 1732277605021Writing region close event to WAL at 1732277605021Closed at 1732277605021 2024-11-22T12:13:25,022 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4f3a8ed48d3b002e0140f5d1f4422c45, NAME => 'test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:25,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741861_1037 (size=45) 2024-11-22T12:13:25,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741861_1037 (size=45) 2024-11-22T12:13:25,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741861_1037 (size=45) 2024-11-22T12:13:25,026 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,026 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b745bd27cbcb9cc89fc04815d334ae24, disabling compactions & flushes 2024-11-22T12:13:25,026 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:25,026 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:25,026 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. after waiting 0 ms 2024-11-22T12:13:25,026 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:25,026 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:25,026 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b745bd27cbcb9cc89fc04815d334ae24: Waiting for close lock at 1732277605026Disabling compacts and flushes for region at 1732277605026Disabling writes for close at 1732277605026Writing region close event to WAL at 1732277605026Closed at 1732277605026 2024-11-22T12:13:25,027 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8a3cd019b15b8a97ff8eeb1bbb051471, NAME => 'test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c 2024-11-22T12:13:25,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741862_1038 (size=45) 2024-11-22T12:13:25,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741862_1038 (size=45) 2024-11-22T12:13:25,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741862_1038 (size=45) 2024-11-22T12:13:25,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4f3a8ed48d3b002e0140f5d1f4422c45, disabling compactions & flushes 2024-11-22T12:13:25,034 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:25,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:25,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. after waiting 0 ms 2024-11-22T12:13:25,035 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:25,035 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:25,035 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4f3a8ed48d3b002e0140f5d1f4422c45: Waiting for close lock at 1732277605034Disabling compacts and flushes for region at 1732277605034Disabling writes for close at 1732277605035 (+1 ms)Writing region close event to WAL at 1732277605035Closed at 1732277605035 2024-11-22T12:13:25,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741863_1039 (size=42) 2024-11-22T12:13:25,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741863_1039 (size=42) 2024-11-22T12:13:25,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741863_1039 (size=42) 2024-11-22T12:13:25,040 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,040 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8a3cd019b15b8a97ff8eeb1bbb051471, disabling compactions & flushes 2024-11-22T12:13:25,040 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:25,040 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:25,040 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. after waiting 0 ms 2024-11-22T12:13:25,041 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:25,041 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:25,041 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8a3cd019b15b8a97ff8eeb1bbb051471: Waiting for close lock at 1732277605040Disabling compacts and flushes for region at 1732277605040Disabling writes for close at 1732277605040Writing region close event to WAL at 1732277605041 (+1 ms)Closed at 1732277605041 2024-11-22T12:13:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T12:13:25,346 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,346 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 794bd231e579bf5991f118c3359c87ab, disabling compactions & flushes 2024-11-22T12:13:25,346 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:25,346 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:25,346 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. after waiting 0 ms 2024-11-22T12:13:25,346 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:25,346 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:25,347 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 794bd231e579bf5991f118c3359c87ab: Waiting for close lock at 1732277605346Disabling compacts and flushes for region at 1732277605346Disabling writes for close at 1732277605346Writing region close event to WAL at 1732277605346Closed at 1732277605346 2024-11-22T12:13:25,350 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T12:13:25,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732277605350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732277605350"}]},"ts":"1732277605350"} 2024-11-22T12:13:25,397 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-22T12:13:25,400 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T12:13:25,403 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732277605401"}]},"ts":"1732277605401"} 2024-11-22T12:13:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T12:13:25,408 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-22T12:13:25,409 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {46e18b47f3d3=0} racks are {/default-rack=0} 2024-11-22T12:13:25,424 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T12:13:25,425 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T12:13:25,425 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-22T12:13:25,425 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T12:13:25,425 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T12:13:25,425 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-22T12:13:25,425 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T12:13:25,425 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T12:13:25,425 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-22T12:13:25,425 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T12:13:25,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6ff71fd9f56108f32cc070df1485ba23, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=142956bd7cdd41ce32216563195a828d, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=800c34543b39c820cefb84a25e72255c, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7aef6551d501af493aaa4fd1077bf9c9, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b4584ab1e18d121599a608cdb4be19ea, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e33fb94d3dd5df2c85edbc81c792b792, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b05283d6644e96595068a6390bc0e1fe, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6c48682b2ea1380365b4d638dd13d2a9, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb5b2e2a4461c2824dc11a5a423a6d19, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=794bd231e579bf5991f118c3359c87ab, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1c19c84c4949f0cb43eed608a20910dc, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88a8bf4fbe7648785154bd80df699f0b, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6dbb25f70e3828779de1b8447a41064c, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cca3907a8067af4c43fffaebaa942770, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7f061dd4aec8e910fcda8dcf65d18b2b, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1716ab8b0523ebd9a28d11dac1fd8e6, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b745bd27cbcb9cc89fc04815d334ae24, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a3cd019b15b8a97ff8eeb1bbb051471, ASSIGN}] 2024-11-22T12:13:25,436 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a3cd019b15b8a97ff8eeb1bbb051471, ASSIGN 2024-11-22T12:13:25,437 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8a3cd019b15b8a97ff8eeb1bbb051471, ASSIGN; state=OFFLINE, location=46e18b47f3d3,38629,1732277601859; forceNewPlan=false, retain=false 2024-11-22T12:13:25,439 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:25,440 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN; state=OFFLINE, location=46e18b47f3d3,37263,1732277602008; forceNewPlan=false, retain=false 2024-11-22T12:13:25,442 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b745bd27cbcb9cc89fc04815d334ae24, ASSIGN 2024-11-22T12:13:25,443 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b745bd27cbcb9cc89fc04815d334ae24, ASSIGN; state=OFFLINE, location=46e18b47f3d3,40513,1732277601956; forceNewPlan=false, retain=false 2024-11-22T12:13:25,445 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1716ab8b0523ebd9a28d11dac1fd8e6, ASSIGN 2024-11-22T12:13:25,447 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d1716ab8b0523ebd9a28d11dac1fd8e6, ASSIGN; state=OFFLINE, location=46e18b47f3d3,38629,1732277601859; forceNewPlan=false, retain=false 2024-11-22T12:13:25,451 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:25,453 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN; state=OFFLINE, location=46e18b47f3d3,37263,1732277602008; forceNewPlan=false, retain=false 2024-11-22T12:13:25,454 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7f061dd4aec8e910fcda8dcf65d18b2b, ASSIGN 2024-11-22T12:13:25,455 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7f061dd4aec8e910fcda8dcf65d18b2b, ASSIGN; state=OFFLINE, location=46e18b47f3d3,40513,1732277601956; forceNewPlan=false, retain=false 2024-11-22T12:13:25,457 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cca3907a8067af4c43fffaebaa942770, ASSIGN 2024-11-22T12:13:25,458 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cca3907a8067af4c43fffaebaa942770, ASSIGN; state=OFFLINE, location=46e18b47f3d3,38629,1732277601859; forceNewPlan=false, retain=false 2024-11-22T12:13:25,459 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:25,460 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN; state=OFFLINE, location=46e18b47f3d3,37263,1732277602008; forceNewPlan=false, retain=false 2024-11-22T12:13:25,462 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6dbb25f70e3828779de1b8447a41064c, ASSIGN 2024-11-22T12:13:25,463 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6dbb25f70e3828779de1b8447a41064c, ASSIGN; state=OFFLINE, location=46e18b47f3d3,38629,1732277601859; forceNewPlan=false, retain=false 2024-11-22T12:13:25,464 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88a8bf4fbe7648785154bd80df699f0b, ASSIGN 2024-11-22T12:13:25,465 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88a8bf4fbe7648785154bd80df699f0b, ASSIGN; state=OFFLINE, location=46e18b47f3d3,40513,1732277601956; forceNewPlan=false, retain=false 2024-11-22T12:13:25,467 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:25,468 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN; state=OFFLINE, location=46e18b47f3d3,37263,1732277602008; forceNewPlan=false, retain=false 2024-11-22T12:13:25,469 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1c19c84c4949f0cb43eed608a20910dc, ASSIGN 2024-11-22T12:13:25,470 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1c19c84c4949f0cb43eed608a20910dc, ASSIGN; state=OFFLINE, location=46e18b47f3d3,40513,1732277601956; forceNewPlan=false, retain=false 2024-11-22T12:13:25,471 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=794bd231e579bf5991f118c3359c87ab, ASSIGN 2024-11-22T12:13:25,473 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=794bd231e579bf5991f118c3359c87ab, ASSIGN; state=OFFLINE, location=46e18b47f3d3,40513,1732277601956; forceNewPlan=false, retain=false 2024-11-22T12:13:25,474 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb5b2e2a4461c2824dc11a5a423a6d19, ASSIGN 2024-11-22T12:13:25,475 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb5b2e2a4461c2824dc11a5a423a6d19, ASSIGN; state=OFFLINE, location=46e18b47f3d3,38629,1732277601859; forceNewPlan=false, retain=false 2024-11-22T12:13:25,477 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:25,478 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN; state=OFFLINE, location=46e18b47f3d3,37263,1732277602008; forceNewPlan=false, retain=false 2024-11-22T12:13:25,479 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6c48682b2ea1380365b4d638dd13d2a9, ASSIGN 2024-11-22T12:13:25,480 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6c48682b2ea1380365b4d638dd13d2a9, ASSIGN; state=OFFLINE, location=46e18b47f3d3,40513,1732277601956; forceNewPlan=false, retain=false 2024-11-22T12:13:25,481 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b05283d6644e96595068a6390bc0e1fe, ASSIGN 2024-11-22T12:13:25,482 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b05283d6644e96595068a6390bc0e1fe, ASSIGN; state=OFFLINE, location=46e18b47f3d3,38629,1732277601859; forceNewPlan=false, retain=false 2024-11-22T12:13:25,483 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:25,485 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN; state=OFFLINE, location=46e18b47f3d3,37263,1732277602008; forceNewPlan=false, retain=false 2024-11-22T12:13:25,486 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e33fb94d3dd5df2c85edbc81c792b792, ASSIGN 2024-11-22T12:13:25,487 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e33fb94d3dd5df2c85edbc81c792b792, ASSIGN; state=OFFLINE, location=46e18b47f3d3,40513,1732277601956; forceNewPlan=false, retain=false 2024-11-22T12:13:25,488 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b4584ab1e18d121599a608cdb4be19ea, ASSIGN 2024-11-22T12:13:25,490 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b4584ab1e18d121599a608cdb4be19ea, ASSIGN; state=OFFLINE, location=46e18b47f3d3,38629,1732277601859; forceNewPlan=false, retain=false 2024-11-22T12:13:25,491 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:25,492 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN; state=OFFLINE, location=46e18b47f3d3,37263,1732277602008; forceNewPlan=false, retain=false 2024-11-22T12:13:25,493 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7aef6551d501af493aaa4fd1077bf9c9, ASSIGN 2024-11-22T12:13:25,494 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7aef6551d501af493aaa4fd1077bf9c9, ASSIGN; state=OFFLINE, location=46e18b47f3d3,38629,1732277601859; forceNewPlan=false, retain=false 2024-11-22T12:13:25,495 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=800c34543b39c820cefb84a25e72255c, ASSIGN 2024-11-22T12:13:25,497 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=800c34543b39c820cefb84a25e72255c, ASSIGN; state=OFFLINE, location=46e18b47f3d3,40513,1732277601956; forceNewPlan=false, retain=false 2024-11-22T12:13:25,498 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:25,501 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN; state=OFFLINE, location=46e18b47f3d3,37263,1732277602008; forceNewPlan=false, retain=false 2024-11-22T12:13:25,503 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=142956bd7cdd41ce32216563195a828d, ASSIGN 2024-11-22T12:13:25,505 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=142956bd7cdd41ce32216563195a828d, ASSIGN; state=OFFLINE, location=46e18b47f3d3,40513,1732277601956; forceNewPlan=false, retain=false 2024-11-22T12:13:25,506 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6ff71fd9f56108f32cc070df1485ba23, ASSIGN 2024-11-22T12:13:25,508 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6ff71fd9f56108f32cc070df1485ba23, ASSIGN; state=OFFLINE, location=46e18b47f3d3,38629,1732277601859; forceNewPlan=false, retain=false 2024-11-22T12:13:25,591 INFO [46e18b47f3d3:35109 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-22T12:13:25,592 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,594 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=88a8bf4fbe7648785154bd80df699f0b, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,594 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=1c19c84c4949f0cb43eed608a20910dc, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,595 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=794bd231e579bf5991f118c3359c87ab, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,596 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b745bd27cbcb9cc89fc04815d334ae24, ASSIGN because future has completed 2024-11-22T12:13:25,598 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure b745bd27cbcb9cc89fc04815d334ae24, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:25,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7f061dd4aec8e910fcda8dcf65d18b2b, ASSIGN because future has completed 2024-11-22T12:13:25,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88a8bf4fbe7648785154bd80df699f0b, ASSIGN because future has completed 2024-11-22T12:13:25,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1c19c84c4949f0cb43eed608a20910dc, ASSIGN because future has completed 2024-11-22T12:13:25,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=794bd231e579bf5991f118c3359c87ab, ASSIGN because future has completed 2024-11-22T12:13:25,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 794bd231e579bf5991f118c3359c87ab, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:25,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6c48682b2ea1380365b4d638dd13d2a9, ASSIGN because future has completed 2024-11-22T12:13:25,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e33fb94d3dd5df2c85edbc81c792b792, ASSIGN because future has completed 2024-11-22T12:13:25,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure e33fb94d3dd5df2c85edbc81c792b792, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:25,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c48682b2ea1380365b4d638dd13d2a9, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:25,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c19c84c4949f0cb43eed608a20910dc, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:25,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a8bf4fbe7648785154bd80df699f0b, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:25,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f061dd4aec8e910fcda8dcf65d18b2b, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:25,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=800c34543b39c820cefb84a25e72255c, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,645 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=142956bd7cdd41ce32216563195a828d, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,647 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,647 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=cca3907a8067af4c43fffaebaa942770, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=800c34543b39c820cefb84a25e72255c, ASSIGN because future has completed 2024-11-22T12:13:25,648 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=6dbb25f70e3828779de1b8447a41064c, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,649 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 800c34543b39c820cefb84a25e72255c, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:25,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=142956bd7cdd41ce32216563195a828d, ASSIGN because future has completed 2024-11-22T12:13:25,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8a3cd019b15b8a97ff8eeb1bbb051471, ASSIGN because future has completed 2024-11-22T12:13:25,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d1716ab8b0523ebd9a28d11dac1fd8e6, ASSIGN because future has completed 2024-11-22T12:13:25,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cca3907a8067af4c43fffaebaa942770, ASSIGN because future has completed 2024-11-22T12:13:25,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure cca3907a8067af4c43fffaebaa942770, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:25,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6dbb25f70e3828779de1b8447a41064c, ASSIGN because future has completed 2024-11-22T12:13:25,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb5b2e2a4461c2824dc11a5a423a6d19, ASSIGN because future has completed 2024-11-22T12:13:25,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb5b2e2a4461c2824dc11a5a423a6d19, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:25,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dbb25f70e3828779de1b8447a41064c, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:25,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1716ab8b0523ebd9a28d11dac1fd8e6, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:25,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a3cd019b15b8a97ff8eeb1bbb051471, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:25,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 142956bd7cdd41ce32216563195a828d, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:25,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=b05283d6644e96595068a6390bc0e1fe, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b4584ab1e18d121599a608cdb4be19ea, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,691 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,691 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ff71fd9f56108f32cc070df1485ba23, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,692 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b05283d6644e96595068a6390bc0e1fe, ASSIGN because future has completed 2024-11-22T12:13:25,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure b05283d6644e96595068a6390bc0e1fe, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:25,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b4584ab1e18d121599a608cdb4be19ea, ASSIGN because future has completed 2024-11-22T12:13:25,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7aef6551d501af493aaa4fd1077bf9c9, ASSIGN because future has completed 2024-11-22T12:13:25,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6ff71fd9f56108f32cc070df1485ba23, ASSIGN because future has completed 2024-11-22T12:13:25,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN because future has completed 2024-11-22T12:13:25,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,37263,1732277602008}] 2024-11-22T12:13:25,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN because future has completed 2024-11-22T12:13:25,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,37263,1732277602008}] 2024-11-22T12:13:25,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ff71fd9f56108f32cc070df1485ba23, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:25,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aef6551d501af493aaa4fd1077bf9c9, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:25,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4584ab1e18d121599a608cdb4be19ea, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:25,722 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,723 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,723 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,724 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,724 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,725 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN because future has completed 2024-11-22T12:13:25,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,37263,1732277602008}] 2024-11-22T12:13:25,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN because future has completed 2024-11-22T12:13:25,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN because future has completed 2024-11-22T12:13:25,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN because future has completed 2024-11-22T12:13:25,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN because future has completed 2024-11-22T12:13:25,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,37263,1732277602008}] 2024-11-22T12:13:25,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN because future has completed 2024-11-22T12:13:25,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cf86cd087fe778d3cb9ffcdb4f32fdf, server=46e18b47f3d3,37263,1732277602008}] 2024-11-22T12:13:25,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,37263,1732277602008}] 2024-11-22T12:13:25,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,37263,1732277602008}] 2024-11-22T12:13:25,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,37263,1732277602008}] 2024-11-22T12:13:25,754 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T12:13:25,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37279, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T12:13:25,762 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:25,762 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => e33fb94d3dd5df2c85edbc81c792b792, NAME => 'test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-22T12:13:25,763 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,763 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,763 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,763 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,765 INFO [StoreOpener-e33fb94d3dd5df2c85edbc81c792b792-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,767 INFO [StoreOpener-e33fb94d3dd5df2c85edbc81c792b792-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e33fb94d3dd5df2c85edbc81c792b792 columnFamilyName family 2024-11-22T12:13:25,767 DEBUG [StoreOpener-e33fb94d3dd5df2c85edbc81c792b792-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,768 INFO [StoreOpener-e33fb94d3dd5df2c85edbc81c792b792-1 {}] regionserver.HStore(327): Store=e33fb94d3dd5df2c85edbc81c792b792/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,768 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,769 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,769 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,770 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,770 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,772 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,775 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/e33fb94d3dd5df2c85edbc81c792b792/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,775 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened e33fb94d3dd5df2c85edbc81c792b792; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68462351, jitterRate=0.020168527960777283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,776 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:25,776 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for e33fb94d3dd5df2c85edbc81c792b792: Running coprocessor pre-open hook at 1732277605763Writing region info on filesystem at 1732277605763Initializing all the Stores at 1732277605764 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605765 (+1 ms)Cleaning up temporary data from old regions at 1732277605770 (+5 ms)Running coprocessor post-open hooks at 1732277605776 (+6 ms)Region opened successfully at 1732277605776 2024-11-22T12:13:25,778 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792., pid=33, masterSystemTime=1732277605753 2024-11-22T12:13:25,781 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:25,781 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:25,781 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:25,782 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 142956bd7cdd41ce32216563195a828d, NAME => 'test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-22T12:13:25,782 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,782 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,782 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,782 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,785 INFO [StoreOpener-142956bd7cdd41ce32216563195a828d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure e33fb94d3dd5df2c85edbc81c792b792, server=46e18b47f3d3,40513,1732277601956 because future has completed 2024-11-22T12:13:25,787 INFO [StoreOpener-142956bd7cdd41ce32216563195a828d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 142956bd7cdd41ce32216563195a828d columnFamilyName family 2024-11-22T12:13:25,788 DEBUG [StoreOpener-142956bd7cdd41ce32216563195a828d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,789 INFO [StoreOpener-142956bd7cdd41ce32216563195a828d-1 {}] regionserver.HStore(327): Store=142956bd7cdd41ce32216563195a828d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,789 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,790 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=12 2024-11-22T12:13:25,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure e33fb94d3dd5df2c85edbc81c792b792, server=46e18b47f3d3,40513,1732277601956 in 170 msec 2024-11-22T12:13:25,791 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,791 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,791 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e33fb94d3dd5df2c85edbc81c792b792, ASSIGN in 364 msec 2024-11-22T12:13:25,794 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,797 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/142956bd7cdd41ce32216563195a828d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,798 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 142956bd7cdd41ce32216563195a828d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72157407, jitterRate=0.07522915303707123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,798 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:25,798 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 142956bd7cdd41ce32216563195a828d: Running coprocessor pre-open hook at 1732277605782Writing region info on filesystem at 1732277605782Initializing all the Stores at 1732277605784 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605784Cleaning up temporary data from old regions at 1732277605791 (+7 ms)Running coprocessor post-open hooks at 1732277605798 (+7 ms)Region opened successfully at 1732277605798 2024-11-22T12:13:25,799 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d., pid=44, masterSystemTime=1732277605753 2024-11-22T12:13:25,801 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:25,802 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:25,802 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:25,802 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f061dd4aec8e910fcda8dcf65d18b2b, NAME => 'test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-22T12:13:25,803 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=142956bd7cdd41ce32216563195a828d, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,803 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,803 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,803 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,803 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,805 INFO [StoreOpener-7f061dd4aec8e910fcda8dcf65d18b2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 142956bd7cdd41ce32216563195a828d, server=46e18b47f3d3,40513,1732277601956 because future has completed 2024-11-22T12:13:25,807 INFO [StoreOpener-7f061dd4aec8e910fcda8dcf65d18b2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f061dd4aec8e910fcda8dcf65d18b2b columnFamilyName family 2024-11-22T12:13:25,808 DEBUG [StoreOpener-7f061dd4aec8e910fcda8dcf65d18b2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,808 INFO [StoreOpener-7f061dd4aec8e910fcda8dcf65d18b2b-1 {}] regionserver.HStore(327): Store=7f061dd4aec8e910fcda8dcf65d18b2b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,809 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,810 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,811 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=6 2024-11-22T12:13:25,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 142956bd7cdd41ce32216563195a828d, server=46e18b47f3d3,40513,1732277601956 in 123 msec 2024-11-22T12:13:25,811 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,811 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,812 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T12:13:25,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58457, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T12:13:25,815 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=142956bd7cdd41ce32216563195a828d, ASSIGN in 384 msec 2024-11-22T12:13:25,819 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7f061dd4aec8e910fcda8dcf65d18b2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,819 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 7f061dd4aec8e910fcda8dcf65d18b2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62750771, jitterRate=-0.06494064629077911}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,819 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:25,820 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 7f061dd4aec8e910fcda8dcf65d18b2b: Running coprocessor pre-open hook at 1732277605803Writing region info on filesystem at 1732277605803Initializing all the Stores at 1732277605805 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605805Cleaning up temporary data from old regions at 1732277605811 (+6 ms)Running coprocessor post-open hooks at 1732277605819 (+8 ms)Region opened successfully at 1732277605820 (+1 ms) 2024-11-22T12:13:25,821 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b., pid=37, masterSystemTime=1732277605753 2024-11-22T12:13:25,824 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:25,824 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 7aef6551d501af493aaa4fd1077bf9c9, NAME => 'test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-22T12:13:25,824 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:25,824 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:25,824 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,824 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,825 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:25,825 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,825 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,825 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 800c34543b39c820cefb84a25e72255c, NAME => 'test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-22T12:13:25,825 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,825 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,825 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,825 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,827 INFO [StoreOpener-7aef6551d501af493aaa4fd1077bf9c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,827 INFO [StoreOpener-800c34543b39c820cefb84a25e72255c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f061dd4aec8e910fcda8dcf65d18b2b, server=46e18b47f3d3,40513,1732277601956 because future has completed 2024-11-22T12:13:25,830 INFO [StoreOpener-7aef6551d501af493aaa4fd1077bf9c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aef6551d501af493aaa4fd1077bf9c9 columnFamilyName family 2024-11-22T12:13:25,830 INFO [StoreOpener-800c34543b39c820cefb84a25e72255c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 800c34543b39c820cefb84a25e72255c columnFamilyName family 2024-11-22T12:13:25,830 DEBUG [StoreOpener-800c34543b39c820cefb84a25e72255c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,830 DEBUG [StoreOpener-7aef6551d501af493aaa4fd1077bf9c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,830 INFO [StoreOpener-7aef6551d501af493aaa4fd1077bf9c9-1 {}] regionserver.HStore(327): Store=7aef6551d501af493aaa4fd1077bf9c9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,830 INFO [StoreOpener-800c34543b39c820cefb84a25e72255c-1 {}] regionserver.HStore(327): Store=800c34543b39c820cefb84a25e72255c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,831 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,831 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,832 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,832 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,832 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,833 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,833 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,834 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,834 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,834 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=25 2024-11-22T12:13:25,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f061dd4aec8e910fcda8dcf65d18b2b, server=46e18b47f3d3,40513,1732277601956 in 191 msec 2024-11-22T12:13:25,835 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,838 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7f061dd4aec8e910fcda8dcf65d18b2b, ASSIGN in 408 msec 2024-11-22T12:13:25,839 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/800c34543b39c820cefb84a25e72255c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,840 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 800c34543b39c820cefb84a25e72255c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74140489, jitterRate=0.10477937757968903}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,840 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:25,840 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 800c34543b39c820cefb84a25e72255c: Running coprocessor pre-open hook at 1732277605825Writing region info on filesystem at 1732277605825Initializing all the Stores at 1732277605827 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605827Cleaning up temporary data from old regions at 1732277605833 (+6 ms)Running coprocessor post-open hooks at 1732277605840 (+7 ms)Region opened successfully at 1732277605840 2024-11-22T12:13:25,841 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7aef6551d501af493aaa4fd1077bf9c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,841 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c., pid=38, masterSystemTime=1732277605753 2024-11-22T12:13:25,841 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 7aef6551d501af493aaa4fd1077bf9c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72102243, jitterRate=0.07440714538097382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,842 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:25,842 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 7aef6551d501af493aaa4fd1077bf9c9: Running coprocessor pre-open hook at 1732277605825Writing region info on filesystem at 1732277605825Initializing all the Stores at 1732277605826 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605826Cleaning up temporary data from old regions at 1732277605834 (+8 ms)Running coprocessor post-open hooks at 1732277605842 (+8 ms)Region opened successfully at 1732277605842 2024-11-22T12:13:25,843 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9., pid=49, masterSystemTime=1732277605811 2024-11-22T12:13:25,846 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:25,846 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:25,847 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=800c34543b39c820cefb84a25e72255c, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,847 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:25,847 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:25,847 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:25,847 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:25,847 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c48682b2ea1380365b4d638dd13d2a9, NAME => 'test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-22T12:13:25,847 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => eb5b2e2a4461c2824dc11a5a423a6d19, NAME => 'test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-22T12:13:25,847 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,847 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,848 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,848 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,848 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,848 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,848 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,848 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 800c34543b39c820cefb84a25e72255c, server=46e18b47f3d3,40513,1732277601956 because future has completed 2024-11-22T12:13:25,850 INFO [StoreOpener-eb5b2e2a4461c2824dc11a5a423a6d19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,850 INFO [StoreOpener-6c48682b2ea1380365b4d638dd13d2a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,852 INFO [StoreOpener-eb5b2e2a4461c2824dc11a5a423a6d19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb5b2e2a4461c2824dc11a5a423a6d19 columnFamilyName family 2024-11-22T12:13:25,853 INFO [StoreOpener-6c48682b2ea1380365b4d638dd13d2a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c48682b2ea1380365b4d638dd13d2a9 columnFamilyName family 2024-11-22T12:13:25,853 DEBUG [StoreOpener-eb5b2e2a4461c2824dc11a5a423a6d19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,853 DEBUG [StoreOpener-6c48682b2ea1380365b4d638dd13d2a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,853 INFO [StoreOpener-eb5b2e2a4461c2824dc11a5a423a6d19-1 {}] regionserver.HStore(327): Store=eb5b2e2a4461c2824dc11a5a423a6d19/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,854 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,854 INFO [StoreOpener-6c48682b2ea1380365b4d638dd13d2a9-1 {}] regionserver.HStore(327): Store=6c48682b2ea1380365b4d638dd13d2a9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,854 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,855 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=46e18b47f3d3,40513,1732277601956, table=test, region=800c34543b39c820cefb84a25e72255c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T12:13:25,855 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,855 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,856 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,857 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,857 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,857 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,858 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,859 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,861 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,863 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,863 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. 2024-11-22T12:13:25,863 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => d5254a456e6265d346df2dc9fbb1afe4, NAME => 'test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-22T12:13:25,864 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,864 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,865 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6c48682b2ea1380365b4d638dd13d2a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,866 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=8 2024-11-22T12:13:25,866 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 800c34543b39c820cefb84a25e72255c, server=46e18b47f3d3,40513,1732277601956 in 209 msec 2024-11-22T12:13:25,866 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 6c48682b2ea1380365b4d638dd13d2a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73865714, jitterRate=0.10068491101264954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,867 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:25,867 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 6c48682b2ea1380365b4d638dd13d2a9: Running coprocessor pre-open hook at 1732277605848Writing region info on filesystem at 1732277605848Initializing all the Stores at 1732277605850 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605850Cleaning up temporary data from old regions at 1732277605857 (+7 ms)Running coprocessor post-open hooks at 1732277605867 (+10 ms)Region opened successfully at 1732277605867 2024-11-22T12:13:25,868 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9., pid=34, masterSystemTime=1732277605753 2024-11-22T12:13:25,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=800c34543b39c820cefb84a25e72255c, ASSIGN in 439 msec 2024-11-22T12:13:25,871 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,871 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:25,871 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:25,871 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:25,872 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 794bd231e579bf5991f118c3359c87ab, NAME => 'test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-22T12:13:25,872 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,872 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,872 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,872 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,873 INFO [StoreOpener-d5254a456e6265d346df2dc9fbb1afe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,873 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/eb5b2e2a4461c2824dc11a5a423a6d19/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,874 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened eb5b2e2a4461c2824dc11a5a423a6d19; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66367161, jitterRate=-0.011052235960960388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,874 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:25,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aef6551d501af493aaa4fd1077bf9c9, server=46e18b47f3d3,38629,1732277601859 because future has completed 2024-11-22T12:13:25,874 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for eb5b2e2a4461c2824dc11a5a423a6d19: Running coprocessor pre-open hook at 1732277605848Writing region info on filesystem at 1732277605848Initializing all the Stores at 1732277605850 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605850Cleaning up temporary data from old regions at 1732277605859 (+9 ms)Running coprocessor post-open hooks at 1732277605874 (+15 ms)Region opened successfully at 1732277605874 2024-11-22T12:13:25,876 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19., pid=40, masterSystemTime=1732277605811 2024-11-22T12:13:25,876 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,876 INFO [StoreOpener-794bd231e579bf5991f118c3359c87ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,876 INFO [StoreOpener-d5254a456e6265d346df2dc9fbb1afe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5254a456e6265d346df2dc9fbb1afe4 columnFamilyName family 2024-11-22T12:13:25,876 DEBUG [StoreOpener-d5254a456e6265d346df2dc9fbb1afe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,878 INFO [StoreOpener-d5254a456e6265d346df2dc9fbb1afe4-1 {}] regionserver.HStore(327): Store=d5254a456e6265d346df2dc9fbb1afe4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,878 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c48682b2ea1380365b4d638dd13d2a9, server=46e18b47f3d3,40513,1732277601956 because future has completed 2024-11-22T12:13:25,880 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,881 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:25,881 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:25,881 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:25,881 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,881 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 6dbb25f70e3828779de1b8447a41064c, NAME => 'test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-22T12:13:25,881 INFO [StoreOpener-794bd231e579bf5991f118c3359c87ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 794bd231e579bf5991f118c3359c87ab columnFamilyName family 2024-11-22T12:13:25,882 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,882 DEBUG [StoreOpener-794bd231e579bf5991f118c3359c87ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,882 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,882 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,882 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,882 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,882 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,883 INFO [StoreOpener-794bd231e579bf5991f118c3359c87ab-1 {}] regionserver.HStore(327): Store=794bd231e579bf5991f118c3359c87ab/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,883 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,884 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,885 INFO [StoreOpener-6dbb25f70e3828779de1b8447a41064c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,885 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,885 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,886 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,886 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,887 INFO [StoreOpener-6dbb25f70e3828779de1b8447a41064c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6dbb25f70e3828779de1b8447a41064c columnFamilyName family 2024-11-22T12:13:25,887 DEBUG [StoreOpener-6dbb25f70e3828779de1b8447a41064c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,888 INFO [StoreOpener-6dbb25f70e3828779de1b8447a41064c-1 {}] regionserver.HStore(327): Store=6dbb25f70e3828779de1b8447a41064c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,888 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=9 2024-11-22T12:13:25,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 7aef6551d501af493aaa4fd1077bf9c9, server=46e18b47f3d3,38629,1732277601859 in 169 msec 2024-11-22T12:13:25,890 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,890 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,890 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,891 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,891 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7aef6551d501af493aaa4fd1077bf9c9, ASSIGN in 462 msec 2024-11-22T12:13:25,894 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=15 2024-11-22T12:13:25,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c48682b2ea1380365b4d638dd13d2a9, server=46e18b47f3d3,40513,1732277601956 in 273 msec 2024-11-22T12:13:25,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6c48682b2ea1380365b4d638dd13d2a9, ASSIGN in 471 msec 2024-11-22T12:13:25,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,905 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6dbb25f70e3828779de1b8447a41064c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,905 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d5254a456e6265d346df2dc9fbb1afe4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,905 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/794bd231e579bf5991f118c3359c87ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,906 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened d5254a456e6265d346df2dc9fbb1afe4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69748417, jitterRate=0.03933240473270416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,906 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:25,906 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for d5254a456e6265d346df2dc9fbb1afe4: Running coprocessor pre-open hook at 1732277605866Writing region info on filesystem at 1732277605866Initializing all the Stores at 1732277605868 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605868Cleaning up temporary data from old regions at 1732277605882 (+14 ms)Running coprocessor post-open hooks at 1732277605906 (+24 ms)Region opened successfully at 1732277605906 2024-11-22T12:13:25,906 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 794bd231e579bf5991f118c3359c87ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64905658, jitterRate=-0.03283032774925232}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,906 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:25,906 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 794bd231e579bf5991f118c3359c87ab: Running coprocessor pre-open hook at 1732277605873Writing region info on filesystem at 1732277605873Initializing all the Stores at 1732277605874 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605875 (+1 ms)Cleaning up temporary data from old regions at 1732277605886 (+11 ms)Running coprocessor post-open hooks at 1732277605906 (+20 ms)Region opened successfully at 1732277605906 2024-11-22T12:13:25,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb5b2e2a4461c2824dc11a5a423a6d19, server=46e18b47f3d3,38629,1732277601859 because future has completed 2024-11-22T12:13:25,906 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 6dbb25f70e3828779de1b8447a41064c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68156433, jitterRate=0.015609994530677795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,907 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:25,907 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 6dbb25f70e3828779de1b8447a41064c: Running coprocessor pre-open hook at 1732277605882Writing region info on filesystem at 1732277605882Initializing all the Stores at 1732277605884 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605884Cleaning up temporary data from old regions at 1732277605891 (+7 ms)Running coprocessor post-open hooks at 1732277605907 (+16 ms)Region opened successfully at 1732277605907 2024-11-22T12:13:25,907 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4., pid=54, masterSystemTime=1732277605854 2024-11-22T12:13:25,907 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab., pid=32, masterSystemTime=1732277605753 2024-11-22T12:13:25,908 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c., pid=41, masterSystemTime=1732277605811 2024-11-22T12:13:25,910 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:25,910 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:25,911 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:25,911 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ff71fd9f56108f32cc070df1485ba23, NAME => 'test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-22T12:13:25,911 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,911 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,912 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,912 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,912 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. 2024-11-22T12:13:25,913 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. 2024-11-22T12:13:25,913 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:25,913 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c61ef28412030a5de1748e0494b5d77, NAME => 'test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-22T12:13:25,913 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,914 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,914 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,914 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,914 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:25,914 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:25,915 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:25,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=17 2024-11-22T12:13:25,915 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c19c84c4949f0cb43eed608a20910dc, NAME => 'test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-22T12:13:25,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure eb5b2e2a4461c2824dc11a5a423a6d19, server=46e18b47f3d3,38629,1732277601859 in 245 msec 2024-11-22T12:13:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T12:13:25,915 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,915 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,916 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,915 INFO [StoreOpener-6ff71fd9f56108f32cc070df1485ba23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,916 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,917 INFO [StoreOpener-4c61ef28412030a5de1748e0494b5d77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,918 INFO [StoreOpener-1c19c84c4949f0cb43eed608a20910dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,918 INFO [StoreOpener-6ff71fd9f56108f32cc070df1485ba23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ff71fd9f56108f32cc070df1485ba23 columnFamilyName family 2024-11-22T12:13:25,918 DEBUG [StoreOpener-6ff71fd9f56108f32cc070df1485ba23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,919 INFO [StoreOpener-6ff71fd9f56108f32cc070df1485ba23-1 {}] regionserver.HStore(327): Store=6ff71fd9f56108f32cc070df1485ba23/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,919 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb5b2e2a4461c2824dc11a5a423a6d19, ASSIGN in 488 msec 2024-11-22T12:13:25,920 INFO [StoreOpener-4c61ef28412030a5de1748e0494b5d77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c61ef28412030a5de1748e0494b5d77 columnFamilyName family 2024-11-22T12:13:25,921 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,920 DEBUG [StoreOpener-4c61ef28412030a5de1748e0494b5d77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,921 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,922 INFO [StoreOpener-4c61ef28412030a5de1748e0494b5d77-1 {}] regionserver.HStore(327): Store=4c61ef28412030a5de1748e0494b5d77/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,922 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,922 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,922 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,923 INFO [StoreOpener-1c19c84c4949f0cb43eed608a20910dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c19c84c4949f0cb43eed608a20910dc columnFamilyName family 2024-11-22T12:13:25,923 DEBUG [StoreOpener-1c19c84c4949f0cb43eed608a20910dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,924 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,925 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,925 INFO [StoreOpener-1c19c84c4949f0cb43eed608a20910dc-1 {}] regionserver.HStore(327): Store=1c19c84c4949f0cb43eed608a20910dc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 794bd231e579bf5991f118c3359c87ab, server=46e18b47f3d3,40513,1732277601956 because future has completed 2024-11-22T12:13:25,925 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,925 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,925 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,926 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,926 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,927 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,927 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,927 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,928 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,37263,1732277602008 because future has completed 2024-11-22T12:13:25,933 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6ff71fd9f56108f32cc070df1485ba23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,934 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4c61ef28412030a5de1748e0494b5d77/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,935 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,935 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 4c61ef28412030a5de1748e0494b5d77; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70338056, jitterRate=0.0481187105178833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,935 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 6ff71fd9f56108f32cc070df1485ba23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68686870, jitterRate=0.023514121770858765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,935 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:25,935 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:25,936 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 4c61ef28412030a5de1748e0494b5d77: Running coprocessor pre-open hook at 1732277605914Writing region info on filesystem at 1732277605914Initializing all the Stores at 1732277605916 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605916Cleaning up temporary data from old regions at 1732277605925 (+9 ms)Running coprocessor post-open hooks at 1732277605935 (+10 ms)Region opened successfully at 1732277605935 2024-11-22T12:13:25,936 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 6ff71fd9f56108f32cc070df1485ba23: Running coprocessor pre-open hook at 1732277605912Writing region info on filesystem at 1732277605912Initializing all the Stores at 1732277605913 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605913Cleaning up temporary data from old regions at 1732277605922 (+9 ms)Running coprocessor post-open hooks at 1732277605935 (+13 ms)Region opened successfully at 1732277605935 2024-11-22T12:13:25,937 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77., pid=51, masterSystemTime=1732277605854 2024-11-22T12:13:25,938 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23., pid=48, masterSystemTime=1732277605811 2024-11-22T12:13:25,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=18 2024-11-22T12:13:25,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 794bd231e579bf5991f118c3359c87ab, server=46e18b47f3d3,40513,1732277601956 in 321 msec 2024-11-22T12:13:25,939 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/1c19c84c4949f0cb43eed608a20910dc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,939 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1c19c84c4949f0cb43eed608a20910dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66452080, jitterRate=-0.009786844253540039}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,940 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:25,940 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1c19c84c4949f0cb43eed608a20910dc: Running coprocessor pre-open hook at 1732277605916Writing region info on filesystem at 1732277605916Initializing all the Stores at 1732277605918 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605918Cleaning up temporary data from old regions at 1732277605927 (+9 ms)Running coprocessor post-open hooks at 1732277605940 (+13 ms)Region opened successfully at 1732277605940 2024-11-22T12:13:25,941 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc., pid=35, masterSystemTime=1732277605753 2024-11-22T12:13:25,941 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:25,941 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:25,942 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:25,942 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => d1e0cdf60c0d514100f271449411b4ed, NAME => 'test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-22T12:13:25,942 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,942 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,942 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,943 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=794bd231e579bf5991f118c3359c87ab, ASSIGN in 511 msec 2024-11-22T12:13:25,945 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:25,945 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:25,945 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:25,946 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => b4584ab1e18d121599a608cdb4be19ea, NAME => 'test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-22T12:13:25,946 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:25,946 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,946 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,946 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,946 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,947 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:25,947 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:25,947 INFO [StoreOpener-d1e0cdf60c0d514100f271449411b4ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,947 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:25,947 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 88a8bf4fbe7648785154bd80df699f0b, NAME => 'test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-22T12:13:25,948 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,948 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,949 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,949 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ff71fd9f56108f32cc070df1485ba23, server=46e18b47f3d3,38629,1732277601859 because future has completed 2024-11-22T12:13:25,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,950 INFO [StoreOpener-d1e0cdf60c0d514100f271449411b4ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1e0cdf60c0d514100f271449411b4ed columnFamilyName family 2024-11-22T12:13:25,951 DEBUG [StoreOpener-d1e0cdf60c0d514100f271449411b4ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,953 INFO [StoreOpener-d1e0cdf60c0d514100f271449411b4ed-1 {}] regionserver.HStore(327): Store=d1e0cdf60c0d514100f271449411b4ed/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,953 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c19c84c4949f0cb43eed608a20910dc, server=46e18b47f3d3,40513,1732277601956 because future has completed 2024-11-22T12:13:25,955 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,957 INFO [StoreOpener-b4584ab1e18d121599a608cdb4be19ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,957 INFO [StoreOpener-88a8bf4fbe7648785154bd80df699f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=5 2024-11-22T12:13:25,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ff71fd9f56108f32cc070df1485ba23, server=46e18b47f3d3,38629,1732277601859 in 244 msec 2024-11-22T12:13:25,959 INFO [StoreOpener-b4584ab1e18d121599a608cdb4be19ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4584ab1e18d121599a608cdb4be19ea columnFamilyName family 2024-11-22T12:13:25,959 DEBUG [StoreOpener-b4584ab1e18d121599a608cdb4be19ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,960 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,961 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,961 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6ff71fd9f56108f32cc070df1485ba23, ASSIGN in 532 msec 2024-11-22T12:13:25,963 INFO [StoreOpener-88a8bf4fbe7648785154bd80df699f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a8bf4fbe7648785154bd80df699f0b columnFamilyName family 2024-11-22T12:13:25,963 DEBUG [StoreOpener-88a8bf4fbe7648785154bd80df699f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,964 INFO [StoreOpener-b4584ab1e18d121599a608cdb4be19ea-1 {}] regionserver.HStore(327): Store=b4584ab1e18d121599a608cdb4be19ea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,964 INFO [StoreOpener-88a8bf4fbe7648785154bd80df699f0b-1 {}] regionserver.HStore(327): Store=88a8bf4fbe7648785154bd80df699f0b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,964 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,964 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,966 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=46e18b47f3d3,37263,1732277602008, table=test, region=d5254a456e6265d346df2dc9fbb1afe4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T12:13:25,967 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,967 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,968 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,968 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,968 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,969 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,969 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,969 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,969 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=19 2024-11-22T12:13:25,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c19c84c4949f0cb43eed608a20910dc, server=46e18b47f3d3,40513,1732277601956 in 336 msec 2024-11-22T12:13:25,972 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,973 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,973 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1e0cdf60c0d514100f271449411b4ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1c19c84c4949f0cb43eed608a20910dc, ASSIGN in 543 msec 2024-11-22T12:13:25,974 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened d1e0cdf60c0d514100f271449411b4ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70454584, jitterRate=0.04985511302947998}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,974 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:25,974 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for d1e0cdf60c0d514100f271449411b4ed: Running coprocessor pre-open hook at 1732277605943Writing region info on filesystem at 1732277605943Initializing all the Stores at 1732277605945 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605945Cleaning up temporary data from old regions at 1732277605961 (+16 ms)Running coprocessor post-open hooks at 1732277605974 (+13 ms)Region opened successfully at 1732277605974 2024-11-22T12:13:25,976 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4c61ef28412030a5de1748e0494b5d77, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,976 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed., pid=52, masterSystemTime=1732277605854 2024-11-22T12:13:25,978 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b4584ab1e18d121599a608cdb4be19ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,979 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:25,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,37263,1732277602008 because future has completed 2024-11-22T12:13:25,979 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened b4584ab1e18d121599a608cdb4be19ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65738157, jitterRate=-0.02042512595653534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,979 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:25,980 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:25,980 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:25,980 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for b4584ab1e18d121599a608cdb4be19ea: Running coprocessor pre-open hook at 1732277605946Writing region info on filesystem at 1732277605946Initializing all the Stores at 1732277605950 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605950Cleaning up temporary data from old regions at 1732277605969 (+19 ms)Running coprocessor post-open hooks at 1732277605980 (+11 ms)Region opened successfully at 1732277605980 2024-11-22T12:13:25,980 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 7cf86cd087fe778d3cb9ffcdb4f32fdf, NAME => 'test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-22T12:13:25,980 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:25,981 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,981 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:25,981 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea., pid=50, masterSystemTime=1732277605811 2024-11-22T12:13:25,981 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:25,981 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88a8bf4fbe7648785154bd80df699f0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:25,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d1e0cdf60c0d514100f271449411b4ed, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:25,982 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 88a8bf4fbe7648785154bd80df699f0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62198175, jitterRate=-0.07317496836185455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:25,982 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:25,982 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 88a8bf4fbe7648785154bd80df699f0b: Running coprocessor pre-open hook at 1732277605949Writing region info on filesystem at 1732277605949Initializing all the Stores at 1732277605951 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605951Cleaning up temporary data from old regions at 1732277605969 (+18 ms)Running coprocessor post-open hooks at 1732277605982 (+13 ms)Region opened successfully at 1732277605982 2024-11-22T12:13:25,985 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b., pid=36, masterSystemTime=1732277605753 2024-11-22T12:13:25,985 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:25,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,37263,1732277602008 because future has completed 2024-11-22T12:13:25,985 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:25,985 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:25,985 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => b05283d6644e96595068a6390bc0e1fe, NAME => 'test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-22T12:13:25,986 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:25,986 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,986 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:25,986 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:25,989 INFO [StoreOpener-7cf86cd087fe778d3cb9ffcdb4f32fdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:25,989 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:25,989 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:25,989 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:25,989 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => b745bd27cbcb9cc89fc04815d334ae24, NAME => 'test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-22T12:13:25,990 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:25,990 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:25,990 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:25,990 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:25,992 INFO [StoreOpener-7cf86cd087fe778d3cb9ffcdb4f32fdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cf86cd087fe778d3cb9ffcdb4f32fdf columnFamilyName family 2024-11-22T12:13:25,992 DEBUG [StoreOpener-7cf86cd087fe778d3cb9ffcdb4f32fdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=23 2024-11-22T12:13:25,993 INFO [StoreOpener-b05283d6644e96595068a6390bc0e1fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:25,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,37263,1732277602008 in 261 msec 2024-11-22T12:13:25,994 INFO [StoreOpener-7cf86cd087fe778d3cb9ffcdb4f32fdf-1 {}] regionserver.HStore(327): Store=7cf86cd087fe778d3cb9ffcdb4f32fdf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,994 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:25,994 INFO [StoreOpener-b745bd27cbcb9cc89fc04815d334ae24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:25,995 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:25,996 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:25,997 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:25,997 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:25,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN in 567 msec 2024-11-22T12:13:25,997 INFO [StoreOpener-b745bd27cbcb9cc89fc04815d334ae24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b745bd27cbcb9cc89fc04815d334ae24 columnFamilyName family 2024-11-22T12:13:25,997 DEBUG [StoreOpener-b745bd27cbcb9cc89fc04815d334ae24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,998 INFO [StoreOpener-b05283d6644e96595068a6390bc0e1fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05283d6644e96595068a6390bc0e1fe columnFamilyName family 2024-11-22T12:13:25,998 DEBUG [StoreOpener-b05283d6644e96595068a6390bc0e1fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:25,999 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:25,999 INFO [StoreOpener-b05283d6644e96595068a6390bc0e1fe-1 {}] regionserver.HStore(327): Store=b05283d6644e96595068a6390bc0e1fe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,999 INFO [StoreOpener-b745bd27cbcb9cc89fc04815d334ae24-1 {}] regionserver.HStore(327): Store=b745bd27cbcb9cc89fc04815d334ae24/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:25,999 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:26,000 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:26,000 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:26,000 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:26,003 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:26,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a8bf4fbe7648785154bd80df699f0b, server=46e18b47f3d3,40513,1732277601956 because future has completed 2024-11-22T12:13:26,005 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:26,005 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:26,005 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7cf86cd087fe778d3cb9ffcdb4f32fdf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,006 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 7cf86cd087fe778d3cb9ffcdb4f32fdf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61982158, jitterRate=-0.07639387249946594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,006 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:26,006 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 7cf86cd087fe778d3cb9ffcdb4f32fdf: Running coprocessor pre-open hook at 1732277605981Writing region info on filesystem at 1732277605981Initializing all the Stores at 1732277605984 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605984Cleaning up temporary data from old regions at 1732277605997 (+13 ms)Running coprocessor post-open hooks at 1732277606006 (+9 ms)Region opened successfully at 1732277606006 2024-11-22T12:13:26,006 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:26,007 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:26,007 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf., pid=53, masterSystemTime=1732277605854 2024-11-22T12:13:26,010 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:26,011 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:26,011 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:26,011 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f3a8ed48d3b002e0140f5d1f4422c45, NAME => 'test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-22T12:13:26,012 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,012 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:26,012 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,012 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=10 2024-11-22T12:13:26,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,37263,1732277602008 in 274 msec 2024-11-22T12:13:26,014 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b745bd27cbcb9cc89fc04815d334ae24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,014 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b05283d6644e96595068a6390bc0e1fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,015 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened b05283d6644e96595068a6390bc0e1fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59790505, jitterRate=-0.10905204713344574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,015 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:26,015 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for b05283d6644e96595068a6390bc0e1fe: Running coprocessor pre-open hook at 1732277605986Writing region info on filesystem at 1732277605986Initializing all the Stores at 1732277605988 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605988Cleaning up temporary data from old regions at 1732277606005 (+17 ms)Running coprocessor post-open hooks at 1732277606015 (+10 ms)Region opened successfully at 1732277606015 2024-11-22T12:13:26,015 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened b745bd27cbcb9cc89fc04815d334ae24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61159970, jitterRate=-0.08864542841911316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,016 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:26,016 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for b745bd27cbcb9cc89fc04815d334ae24: Running coprocessor pre-open hook at 1732277605990Writing region info on filesystem at 1732277605990Initializing all the Stores at 1732277605992 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277605992Cleaning up temporary data from old regions at 1732277606001 (+9 ms)Running coprocessor post-open hooks at 1732277606016 (+15 ms)Region opened successfully at 1732277606016 2024-11-22T12:13:26,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN in 586 msec 2024-11-22T12:13:26,016 INFO [StoreOpener-4f3a8ed48d3b002e0140f5d1f4422c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,017 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe., pid=45, masterSystemTime=1732277605811 2024-11-22T12:13:26,018 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24., pid=31, masterSystemTime=1732277605753 2024-11-22T12:13:26,018 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:26,021 INFO [StoreOpener-4f3a8ed48d3b002e0140f5d1f4422c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f3a8ed48d3b002e0140f5d1f4422c45 columnFamilyName family 2024-11-22T12:13:26,021 DEBUG [StoreOpener-4f3a8ed48d3b002e0140f5d1f4422c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:26,022 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:26,023 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:26,023 INFO [StoreOpener-4f3a8ed48d3b002e0140f5d1f4422c45-1 {}] regionserver.HStore(327): Store=4f3a8ed48d3b002e0140f5d1f4422c45/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:26,023 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,024 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:26,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cf86cd087fe778d3cb9ffcdb4f32fdf, server=46e18b47f3d3,37263,1732277602008 because future has completed 2024-11-22T12:13:26,025 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,026 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:26,026 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:26,026 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:26,026 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,026 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => d1716ab8b0523ebd9a28d11dac1fd8e6, NAME => 'test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-22T12:13:26,027 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,027 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:26,027 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,027 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,028 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,028 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure b745bd27cbcb9cc89fc04815d334ae24, server=46e18b47f3d3,40513,1732277601956 because future has completed 2024-11-22T12:13:26,030 INFO [StoreOpener-d1716ab8b0523ebd9a28d11dac1fd8e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,031 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:26,034 INFO [StoreOpener-d1716ab8b0523ebd9a28d11dac1fd8e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1716ab8b0523ebd9a28d11dac1fd8e6 columnFamilyName family 2024-11-22T12:13:26,034 DEBUG [StoreOpener-d1716ab8b0523ebd9a28d11dac1fd8e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:26,035 INFO [StoreOpener-d1716ab8b0523ebd9a28d11dac1fd8e6-1 {}] regionserver.HStore(327): Store=d1716ab8b0523ebd9a28d11dac1fd8e6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:26,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure b05283d6644e96595068a6390bc0e1fe, server=46e18b47f3d3,38629,1732277601859 because future has completed 2024-11-22T12:13:26,036 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,037 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4f3a8ed48d3b002e0140f5d1f4422c45/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,037 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,038 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 4f3a8ed48d3b002e0140f5d1f4422c45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70949036, jitterRate=0.05722302198410034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,038 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:26,038 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,038 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 4f3a8ed48d3b002e0140f5d1f4422c45: Running coprocessor pre-open hook at 1732277606012Writing region info on filesystem at 1732277606012Initializing all the Stores at 1732277606014 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277606014Cleaning up temporary data from old regions at 1732277606028 (+14 ms)Running coprocessor post-open hooks at 1732277606038 (+10 ms)Region opened successfully at 1732277606038 2024-11-22T12:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,040 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45., pid=46, masterSystemTime=1732277605854 2024-11-22T12:13:26,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-22T12:13:26,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure b745bd27cbcb9cc89fc04815d334ae24, server=46e18b47f3d3,40513,1732277601956 in 439 msec 2024-11-22T12:13:26,043 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,043 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:26,044 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:26,044 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:26,045 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 88ee6f91206e330acd4027ac1712c4b5, NAME => 'test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-22T12:13:26,045 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:26,045 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,045 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:26,045 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,045 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,37263,1732277602008 because future has completed 2024-11-22T12:13:26,050 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1716ab8b0523ebd9a28d11dac1fd8e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,051 INFO [StoreOpener-88ee6f91206e330acd4027ac1712c4b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,051 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened d1716ab8b0523ebd9a28d11dac1fd8e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74422109, jitterRate=0.1089758425951004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,051 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:26,052 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for d1716ab8b0523ebd9a28d11dac1fd8e6: Running coprocessor pre-open hook at 1732277606028Writing region info on filesystem at 1732277606028Initializing all the Stores at 1732277606030 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277606030Cleaning up temporary data from old regions at 1732277606039 (+9 ms)Running coprocessor post-open hooks at 1732277606051 (+12 ms)Region opened successfully at 1732277606051 2024-11-22T12:13:26,053 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6., pid=42, masterSystemTime=1732277605811 2024-11-22T12:13:26,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b745bd27cbcb9cc89fc04815d334ae24, ASSIGN in 622 msec 2024-11-22T12:13:26,055 INFO [StoreOpener-88ee6f91206e330acd4027ac1712c4b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88ee6f91206e330acd4027ac1712c4b5 columnFamilyName family 2024-11-22T12:13:26,055 DEBUG [StoreOpener-88ee6f91206e330acd4027ac1712c4b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:26,056 INFO [StoreOpener-88ee6f91206e330acd4027ac1712c4b5-1 {}] regionserver.HStore(327): Store=88ee6f91206e330acd4027ac1712c4b5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:26,057 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,057 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:26,057 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:26,057 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:26,057 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => cca3907a8067af4c43fffaebaa942770, NAME => 'test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-22T12:13:26,058 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,058 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:26,058 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,058 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,058 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,059 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,060 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,060 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,060 INFO [StoreOpener-cca3907a8067af4c43fffaebaa942770-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=29 2024-11-22T12:13:26,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,37263,1732277602008 in 354 msec 2024-11-22T12:13:26,063 INFO [StoreOpener-cca3907a8067af4c43fffaebaa942770-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cca3907a8067af4c43fffaebaa942770 columnFamilyName family 2024-11-22T12:13:26,063 DEBUG [StoreOpener-cca3907a8067af4c43fffaebaa942770-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:26,063 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,065 INFO [StoreOpener-cca3907a8067af4c43fffaebaa942770-1 {}] regionserver.HStore(327): Store=cca3907a8067af4c43fffaebaa942770/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:26,065 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,066 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=test, region=b05283d6644e96595068a6390bc0e1fe. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T12:13:26,069 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN in 636 msec 2024-11-22T12:13:26,070 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=46e18b47f3d3,40513,1732277601956, table=test, region=88a8bf4fbe7648785154bd80df699f0b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T12:13:26,071 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:26,072 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=46e18b47f3d3,37263,1732277602008, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T12:13:26,072 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,072 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,072 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=46e18b47f3d3,37263,1732277602008, table=test, region=d5254a456e6265d346df2dc9fbb1afe4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T12:13:26,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1716ab8b0523ebd9a28d11dac1fd8e6, server=46e18b47f3d3,38629,1732277601859 because future has completed 2024-11-22T12:13:26,076 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,077 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88ee6f91206e330acd4027ac1712c4b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,079 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 88ee6f91206e330acd4027ac1712c4b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72806764, jitterRate=0.08490532636642456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,079 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:26,079 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 88ee6f91206e330acd4027ac1712c4b5: Running coprocessor pre-open hook at 1732277606046Writing region info on filesystem at 1732277606046Initializing all the Stores at 1732277606050 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277606050Cleaning up temporary data from old regions at 1732277606060 (+10 ms)Running coprocessor post-open hooks at 1732277606079 (+19 ms)Region opened successfully at 1732277606079 2024-11-22T12:13:26,080 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5., pid=47, masterSystemTime=1732277605854 2024-11-22T12:13:26,080 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/cca3907a8067af4c43fffaebaa942770/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,082 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened cca3907a8067af4c43fffaebaa942770; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71925723, jitterRate=0.07177679240703583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,082 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:26,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=27 2024-11-22T12:13:26,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure d1716ab8b0523ebd9a28d11dac1fd8e6, server=46e18b47f3d3,38629,1732277601859 in 403 msec 2024-11-22T12:13:26,082 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for cca3907a8067af4c43fffaebaa942770: Running coprocessor pre-open hook at 1732277606058Writing region info on filesystem at 1732277606058Initializing all the Stores at 1732277606060 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277606060Cleaning up temporary data from old regions at 1732277606072 (+12 ms)Running coprocessor post-open hooks at 1732277606082 (+10 ms)Region opened successfully at 1732277606082 2024-11-22T12:13:26,084 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770., pid=39, masterSystemTime=1732277605811 2024-11-22T12:13:26,085 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:26,085 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:26,085 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. 2024-11-22T12:13:26,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d1716ab8b0523ebd9a28d11dac1fd8e6, ASSIGN in 655 msec 2024-11-22T12:13:26,085 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => c3b4530e36a471e0bfd4c64c45ec26d5, NAME => 'test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-22T12:13:26,086 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,086 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:26,086 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,086 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,087 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=88ee6f91206e330acd4027ac1712c4b5, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:26,087 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:26,087 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:26,087 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:26,087 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a3cd019b15b8a97ff8eeb1bbb051471, NAME => 'test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-22T12:13:26,088 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,088 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:26,088 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,088 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,088 INFO [StoreOpener-c3b4530e36a471e0bfd4c64c45ec26d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,37263,1732277602008 because future has completed 2024-11-22T12:13:26,091 INFO [StoreOpener-c3b4530e36a471e0bfd4c64c45ec26d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3b4530e36a471e0bfd4c64c45ec26d5 columnFamilyName family 2024-11-22T12:13:26,091 INFO [StoreOpener-8a3cd019b15b8a97ff8eeb1bbb051471-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,091 DEBUG [StoreOpener-c3b4530e36a471e0bfd4c64c45ec26d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:26,092 INFO [StoreOpener-c3b4530e36a471e0bfd4c64c45ec26d5-1 {}] regionserver.HStore(327): Store=c3b4530e36a471e0bfd4c64c45ec26d5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:26,092 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,093 INFO [StoreOpener-8a3cd019b15b8a97ff8eeb1bbb051471-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a3cd019b15b8a97ff8eeb1bbb051471 columnFamilyName family 2024-11-22T12:13:26,093 DEBUG [StoreOpener-8a3cd019b15b8a97ff8eeb1bbb051471-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:26,093 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:26,093 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,093 INFO [StoreOpener-8a3cd019b15b8a97ff8eeb1bbb051471-1 {}] regionserver.HStore(327): Store=8a3cd019b15b8a97ff8eeb1bbb051471/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:26,093 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,094 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,096 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,096 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure cca3907a8067af4c43fffaebaa942770, server=46e18b47f3d3,38629,1732277601859 because future has completed 2024-11-22T12:13:26,097 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,098 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,100 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/c3b4530e36a471e0bfd4c64c45ec26d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,101 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened c3b4530e36a471e0bfd4c64c45ec26d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66600544, jitterRate=-0.007574558258056641}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,102 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/8a3cd019b15b8a97ff8eeb1bbb051471/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,102 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:26,102 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for c3b4530e36a471e0bfd4c64c45ec26d5: Running coprocessor pre-open hook at 1732277606086Writing region info on filesystem at 1732277606086Initializing all the Stores at 1732277606088 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277606088Cleaning up temporary data from old regions at 1732277606095 (+7 ms)Running coprocessor post-open hooks at 1732277606102 (+7 ms)Region opened successfully at 1732277606102 2024-11-22T12:13:26,103 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 8a3cd019b15b8a97ff8eeb1bbb051471; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66637718, jitterRate=-0.007020622491836548}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,103 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:26,103 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5., pid=55, masterSystemTime=1732277605854 2024-11-22T12:13:26,103 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 8a3cd019b15b8a97ff8eeb1bbb051471: Running coprocessor pre-open hook at 1732277606088Writing region info on filesystem at 1732277606088Initializing all the Stores at 1732277606090 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277606090Cleaning up temporary data from old regions at 1732277606096 (+6 ms)Running coprocessor post-open hooks at 1732277606103 (+7 ms)Region opened successfully at 1732277606103 2024-11-22T12:13:26,105 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471., pid=43, masterSystemTime=1732277605811 2024-11-22T12:13:26,105 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. 2024-11-22T12:13:26,106 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. 2024-11-22T12:13:26,106 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:26,106 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 5acff54db0b679ccee30c4cfa789fe5b, NAME => 'test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-22T12:13:26,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=26 2024-11-22T12:13:26,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,37263,1732277602008 in 397 msec 2024-11-22T12:13:26,106 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,106 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:26,107 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,107 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,108 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:26,108 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:26,109 INFO [StoreOpener-5acff54db0b679ccee30c4cfa789fe5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN in 679 msec 2024-11-22T12:13:26,111 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:26,111 INFO [StoreOpener-5acff54db0b679ccee30c4cfa789fe5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5acff54db0b679ccee30c4cfa789fe5b columnFamilyName family 2024-11-22T12:13:26,111 DEBUG [StoreOpener-5acff54db0b679ccee30c4cfa789fe5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:26,112 INFO [StoreOpener-5acff54db0b679ccee30c4cfa789fe5b-1 {}] regionserver.HStore(327): Store=5acff54db0b679ccee30c4cfa789fe5b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:26,112 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,114 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a3cd019b15b8a97ff8eeb1bbb051471, server=46e18b47f3d3,38629,1732277601859 because future has completed 2024-11-22T12:13:26,114 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,115 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,115 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:26,115 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,118 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,37263,1732277602008 because future has completed 2024-11-22T12:13:26,121 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/5acff54db0b679ccee30c4cfa789fe5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T12:13:26,122 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 5acff54db0b679ccee30c4cfa789fe5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73743505, jitterRate=0.09886385500431061}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:26,122 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:26,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=30 2024-11-22T12:13:26,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a3cd019b15b8a97ff8eeb1bbb051471, server=46e18b47f3d3,38629,1732277601859 in 439 msec 2024-11-22T12:13:26,122 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 5acff54db0b679ccee30c4cfa789fe5b: Running coprocessor pre-open hook at 1732277606107Writing region info on filesystem at 1732277606107Initializing all the Stores at 1732277606108 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277606108Cleaning up temporary data from old regions at 1732277606115 (+7 ms)Running coprocessor post-open hooks at 1732277606122 (+7 ms)Region opened successfully at 1732277606122 2024-11-22T12:13:26,123 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b., pid=56, masterSystemTime=1732277605854 2024-11-22T12:13:26,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8a3cd019b15b8a97ff8eeb1bbb051471, ASSIGN in 695 msec 2024-11-22T12:13:26,126 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:26,126 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:26,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:26,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,37263,1732277602008 because future has completed 2024-11-22T12:13:26,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=16 2024-11-22T12:13:26,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,37263,1732277602008 in 384 msec 2024-11-22T12:13:26,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN in 705 msec 2024-11-22T12:13:26,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=20 2024-11-22T12:13:26,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,37263,1732277602008 in 383 msec 2024-11-22T12:13:26,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN in 710 msec 2024-11-22T12:13:26,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-11-22T12:13:26,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure cca3907a8067af4c43fffaebaa942770, server=46e18b47f3d3,38629,1732277601859 in 482 msec 2024-11-22T12:13:26,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca3907a8067af4c43fffaebaa942770, ASSIGN in 717 msec 2024-11-22T12:13:26,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=14 2024-11-22T12:13:26,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure b05283d6644e96595068a6390bc0e1fe, server=46e18b47f3d3,38629,1732277601859 in 454 msec 2024-11-22T12:13:26,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b05283d6644e96595068a6390bc0e1fe, ASSIGN in 722 msec 2024-11-22T12:13:26,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-11-22T12:13:26,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7cf86cd087fe778d3cb9ffcdb4f32fdf, server=46e18b47f3d3,37263,1732277602008 in 417 msec 2024-11-22T12:13:26,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN in 729 msec 2024-11-22T12:13:26,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=21 2024-11-22T12:13:26,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 88a8bf4fbe7648785154bd80df699f0b, server=46e18b47f3d3,40513,1732277601956 in 526 msec 2024-11-22T12:13:26,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=88a8bf4fbe7648785154bd80df699f0b, ASSIGN in 733 msec 2024-11-22T12:13:26,163 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:26,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4584ab1e18d121599a608cdb4be19ea, server=46e18b47f3d3,38629,1732277601859 because future has completed 2024-11-22T12:13:26,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=13 2024-11-22T12:13:26,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,37263,1732277602008 in 424 msec 2024-11-22T12:13:26,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN in 741 msec 2024-11-22T12:13:26,173 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=test, region=b4584ab1e18d121599a608cdb4be19ea. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T12:13:26,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=11 2024-11-22T12:13:26,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure b4584ab1e18d121599a608cdb4be19ea, server=46e18b47f3d3,38629,1732277601859 in 455 msec 2024-11-22T12:13:26,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b4584ab1e18d121599a608cdb4be19ea, ASSIGN in 752 msec 2024-11-22T12:13:26,182 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, openSeqNum=2, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:26,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dbb25f70e3828779de1b8447a41064c, server=46e18b47f3d3,38629,1732277601859 because future has completed 2024-11-22T12:13:26,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=22 2024-11-22T12:13:26,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 6dbb25f70e3828779de1b8447a41064c, server=46e18b47f3d3,38629,1732277601859 in 517 msec 2024-11-22T12:13:26,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=4 2024-11-22T12:13:26,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6dbb25f70e3828779de1b8447a41064c, ASSIGN in 762 msec 2024-11-22T12:13:26,194 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T12:13:26,195 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732277606195"}]},"ts":"1732277606195"} 2024-11-22T12:13:26,198 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-22T12:13:26,200 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T12:13:26,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4380 sec 2024-11-22T12:13:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T12:13:26,941 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-22T12:13:26,941 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-22T12:13:26,943 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T12:13:26,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-22T12:13:26,968 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T12:13:26,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-22T12:13:26,969 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T12:13:26,976 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-22T12:13:27,003 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-22T12:13:27,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T12:13:27,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T12:13:27,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T12:13:27,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T12:13:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7cf86cd087fe778d3cb9ffcdb4f32fdf changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d1e0cdf60c0d514100f271449411b4ed changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d5254a456e6265d346df2dc9fbb1afe4 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c3b4530e36a471e0bfd4c64c45ec26d5 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5acff54db0b679ccee30c4cfa789fe5b changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4c61ef28412030a5de1748e0494b5d77 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 88ee6f91206e330acd4027ac1712c4b5 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4f3a8ed48d3b002e0140f5d1f4422c45 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6ff71fd9f56108f32cc070df1485ba23 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7aef6551d501af493aaa4fd1077bf9c9 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b4584ab1e18d121599a608cdb4be19ea changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b05283d6644e96595068a6390bc0e1fe changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eb5b2e2a4461c2824dc11a5a423a6d19 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6dbb25f70e3828779de1b8447a41064c changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cca3907a8067af4c43fffaebaa942770 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d1716ab8b0523ebd9a28d11dac1fd8e6 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8a3cd019b15b8a97ff8eeb1bbb051471 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 142956bd7cdd41ce32216563195a828d changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 800c34543b39c820cefb84a25e72255c changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e33fb94d3dd5df2c85edbc81c792b792 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6c48682b2ea1380365b4d638dd13d2a9 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 794bd231e579bf5991f118c3359c87ab changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1c19c84c4949f0cb43eed608a20910dc changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 88a8bf4fbe7648785154bd80df699f0b changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7f061dd4aec8e910fcda8dcf65d18b2b changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b745bd27cbcb9cc89fc04815d334ae24 changed from -1.0 to 0.0, refreshing cache 2024-11-22T12:13:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-22T12:13:27,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-22T12:13:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-22T12:13:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(204): Hosts are {46e18b47f3d3=0} racks are {/default-rack=0} 2024-11-22T12:13:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-22T12:13:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-22T12:13:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-22T12:13:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T12:13:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T12:13:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-22T12:13:27,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T12:13:27,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T12:13:27,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-22T12:13:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-22T12:13:27,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-22T12:13:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-22T12:13:27,096 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T12:13:27,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-22T12:13:27,126 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=380, OpenFileDescriptor=638, MaxFileDescriptor=1048576, SystemLoadAverage=328, ProcessCount=11, AvailableMemoryMB=2663 2024-11-22T12:13:27,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-22T12:13:27,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-22T12:13:27,131 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:27,131 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 46e18b47f3d3,37263,1732277602008: Simulated kill ***** 2024-11-22T12:13:27,131 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-22T12:13:27,133 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-22T12:13:27,136 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-22T12:13:27,136 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-22T12:13:27,136 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-22T12:13:27,136 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 301915728 }, "NonHeapMemoryUsage": { "committed": 152174592, "init": 7667712, "max": -1, "used": 149774576 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "46e18b47f3d3" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "46e18b47f3d3" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "46e18b47f3d3" } ] } 2024-11-22T12:13:27,139 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35109 {}] master.MasterRpcServices(700): 46e18b47f3d3,37263,1732277602008 reported a fatal error: ***** ABORTING region server 46e18b47f3d3,37263,1732277602008: Simulated kill ***** 2024-11-22T12:13:27,142 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '46e18b47f3d3,37263,1732277602008' ***** 2024-11-22T12:13:27,142 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-22T12:13:27,142 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T12:13:27,143 INFO [RS:2;46e18b47f3d3:37263 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-22T12:13:27,143 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T12:13:27,143 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T12:13:27,143 INFO [RS:2;46e18b47f3d3:37263 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-22T12:13:27,143 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T12:13:27,143 INFO [RS:2;46e18b47f3d3:37263 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;46e18b47f3d3:37263. 2024-11-22T12:13:27,143 DEBUG [RS:2;46e18b47f3d3:37263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T12:13:27,152 DEBUG [RS:2;46e18b47f3d3:37263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:27,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741836_1012 (size=32114) 2024-11-22T12:13:27,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741836_1012 (size=32114) 2024-11-22T12:13:27,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741836_1012 (size=32114) 2024-11-22T12:13:27,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741833_1009 (size=2237) 2024-11-22T12:13:27,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741833_1009 (size=2237) 2024-11-22T12:13:27,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741833_1009 (size=2237) 2024-11-22T12:13:27,165 DEBUG [RS:2;46e18b47f3d3:37263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:27,165 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T12:13:27,165 INFO [RS:2;46e18b47f3d3:37263 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37263 2024-11-22T12:13:27,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/46e18b47f3d3,37263,1732277602008 2024-11-22T12:13:27,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T12:13:27,233 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T12:13:27,239 INFO [regionserver/46e18b47f3d3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T12:13:27,244 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [46e18b47f3d3,37263,1732277602008] 2024-11-22T12:13:27,254 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/46e18b47f3d3,37263,1732277602008 already deleted, retry=false 2024-11-22T12:13:27,254 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 46e18b47f3d3,37263,1732277602008 on 46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:27,258 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:27,263 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 46e18b47f3d3,37263,1732277602008 (carryingMeta=true) 46e18b47f3d3,37263,1732277602008/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@46ff4d07[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-22T12:13:27,263 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:27,265 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true, isMeta: true 2024-11-22T12:13:27,267 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting 2024-11-22T12:13:27,269 INFO [PEWorker-1 {}] master.SplitWALManager(105): 46e18b47f3d3,37263,1732277602008 WAL count=1, meta=true 2024-11-22T12:13:27,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta}] 2024-11-22T12:13:27,276 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:27,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:27,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T12:13:27,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37263-0x10162767bb80003, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T12:13:27,345 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T12:13:27,345 INFO [RS:2;46e18b47f3d3:37263 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T12:13:27,345 INFO [RS:2;46e18b47f3d3:37263 {}] regionserver.HRegionServer(1031): Exiting; stopping=46e18b47f3d3,37263,1732277602008; zookeeper connection closed. 2024-11-22T12:13:27,346 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66ec1b17 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66ec1b17 2024-11-22T12:13:27,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-22T12:13:27,517 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, size=31.4 K (32114bytes) 2024-11-22T12:13:27,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-22T12:13:27,519 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-22T12:13:27,519 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta 2024-11-22T12:13:27,528 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta after 5ms 2024-11-22T12:13:27,536 INFO [regionserver/46e18b47f3d3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T12:13:27,536 INFO [regionserver/46e18b47f3d3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T12:13:27,540 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta took 33ms 2024-11-22T12:13:27,571 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-22T12:13:27,578 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta.temp 2024-11-22T12:13:27,584 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta.temp 2024-11-22T12:13:27,586 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta so closing down 2024-11-22T12:13:27,586 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-22T12:13:27,589 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-22T12:13:27,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741864_1040 (size=31873) 2024-11-22T12:13:27,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741864_1040 (size=31873) 2024-11-22T12:13:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741864_1040 (size=31873) 2024-11-22T12:13:27,634 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-22T12:13:27,637 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta.temp to hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-22T12:13:27,643 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 84 ms; skipped=1; WAL=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-22T12:13:27,643 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, journal: Splitting hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, size=31.4 K (32114bytes) at 1732277607517Creating recovered edits writer path=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta.temp at 1732277607584 (+67 ms)Finishing writing output for hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta so closing down at 1732277607586 (+2 ms)3 split writer threads finished at 1732277607589 (+3 ms)Closed recovered edits writer path=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1732277607634 (+45 ms)Rename recovered edits hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta.temp to hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732277607637 (+3 ms)Processed 58 edits across 1 Regions in 84 ms; skipped=1; WAL=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1732277607643 (+6 ms) 2024-11-22T12:13:27,643 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta 2024-11-22T12:13:27,645 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-22T12:13:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-22T12:13:27,657 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta to hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs 2024-11-22T12:13:27,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-22T12:13:27,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 in 379 msec 2024-11-22T12:13:27,668 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:27,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-22T12:13:27,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 in 397 msec 2024-11-22T12:13:27,677 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting dir is empty, no logs to split. 2024-11-22T12:13:27,677 INFO [PEWorker-1 {}] master.SplitWALManager(105): 46e18b47f3d3,37263,1732277602008 WAL count=0, meta=true 2024-11-22T12:13:27,677 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 46e18b47f3d3,37263,1732277602008 WAL splitting is done? wals=0, meta=true 2024-11-22T12:13:27,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T12:13:27,682 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:27,683 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T12:13:27,833 DEBUG [46e18b47f3d3:35109 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T12:13:27,834 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(204): Hosts are {46e18b47f3d3=0} racks are {/default-rack=0} 2024-11-22T12:13:27,834 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T12:13:27,834 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T12:13:27,834 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T12:13:27,834 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T12:13:27,834 INFO [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T12:13:27,835 INFO [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T12:13:27,835 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T12:13:27,835 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:27,836 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPENING 2024-11-22T12:13:27,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:27,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:27,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:27,896 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:27,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:27,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:27,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:27,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:28,057 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T12:13:28,057 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T12:13:28,057 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T12:13:28,060 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=46e18b47f3d3%2C38629%2C1732277601859.meta, suffix=.meta, logDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,38629,1732277601859, archiveDir=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs, maxLogs=32 2024-11-22T12:13:28,073 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,38629,1732277601859/46e18b47f3d3%2C38629%2C1732277601859.meta.1732277608061.meta, exclude list is [], retry=0 2024-11-22T12:13:28,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-0d6673b4-5e58-4255-9105-109b61968622,DISK] 2024-11-22T12:13:28,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-65ff8120-9702-4fea-8011-038991f8f8f0,DISK] 2024-11-22T12:13:28,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33883,DS-1b13ec14-000c-4db2-b97b-89138789e6a7,DISK] 2024-11-22T12:13:28,079 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,38629,1732277601859/46e18b47f3d3%2C38629%2C1732277601859.meta.1732277608061.meta 2024-11-22T12:13:28,079 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:44637:44637),(127.0.0.1/127.0.0.1:34481:34481)] 2024-11-22T12:13:28,080 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T12:13:28,080 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T12:13:28,080 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T12:13:28,080 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T12:13:28,080 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T12:13:28,080 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:28,080 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T12:13:28,081 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T12:13:28,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T12:13:28,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T12:13:28,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:28,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:28,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T12:13:28,086 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T12:13:28,086 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:28,087 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:28,087 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T12:13:28,088 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T12:13:28,088 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:28,089 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:28,089 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T12:13:28,090 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T12:13:28,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:28,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T12:13:28,091 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T12:13:28,093 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740 2024-11-22T12:13:28,096 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740 2024-11-22T12:13:28,097 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-22T12:13:28,120 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-22T12:13:28,123 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-22T12:13:28,192 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/info/d65b7cdabcb14cd0be7060b8c747f87d is 138, key is test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d./info:regioninfo/1732277605802/Put/seqid=0 2024-11-22T12:13:28,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741866_1042 (size=34924) 2024-11-22T12:13:28,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741866_1042 (size=34924) 2024-11-22T12:13:28,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741866_1042 (size=34924) 2024-11-22T12:13:28,208 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/info/d65b7cdabcb14cd0be7060b8c747f87d 2024-11-22T12:13:28,266 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/ns/f317ff9661144bb1b3828348ad523e89 is 43, key is default/ns:d/1732277604501/Put/seqid=0 2024-11-22T12:13:28,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741867_1043 (size=5153) 2024-11-22T12:13:28,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741867_1043 (size=5153) 2024-11-22T12:13:28,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741867_1043 (size=5153) 2024-11-22T12:13:28,274 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/ns/f317ff9661144bb1b3828348ad523e89 2024-11-22T12:13:28,298 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/table/f273e4f7e139476da10397cc1f0e4b6f is 40, key is test/table:state/1732277606195/Put/seqid=0 2024-11-22T12:13:28,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741868_1044 (size=5165) 2024-11-22T12:13:28,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741868_1044 (size=5165) 2024-11-22T12:13:28,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741868_1044 (size=5165) 2024-11-22T12:13:28,312 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/table/f273e4f7e139476da10397cc1f0e4b6f 2024-11-22T12:13:28,322 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/info/d65b7cdabcb14cd0be7060b8c747f87d as hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/info/d65b7cdabcb14cd0be7060b8c747f87d 2024-11-22T12:13:28,332 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/info/d65b7cdabcb14cd0be7060b8c747f87d, entries=260, sequenceid=60, filesize=34.1 K 2024-11-22T12:13:28,336 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/ns/f317ff9661144bb1b3828348ad523e89 as hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/ns/f317ff9661144bb1b3828348ad523e89 2024-11-22T12:13:28,345 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/ns/f317ff9661144bb1b3828348ad523e89, entries=2, sequenceid=60, filesize=5.0 K 2024-11-22T12:13:28,347 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/table/f273e4f7e139476da10397cc1f0e4b6f as hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/table/f273e4f7e139476da10397cc1f0e4b6f 2024-11-22T12:13:28,357 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/table/f273e4f7e139476da10397cc1f0e4b6f, entries=2, sequenceid=60, filesize=5.0 K 2024-11-22T12:13:28,363 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 237ms, sequenceid=60, compaction requested=false; wal=null 2024-11-22T12:13:28,364 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T12:13:28,368 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-22T12:13:28,370 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T12:13:28,370 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T12:13:28,371 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T12:13:28,374 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T12:13:28,377 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-22T12:13:28,378 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70303561, jitterRate=0.047604694962501526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T12:13:28,378 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T12:13:28,379 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732277608081Writing region info on filesystem at 1732277608081Initializing all the Stores at 1732277608082 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277608082Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277608082Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277608082Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732277608082Obtaining lock to block concurrent updates at 1732277608123 (+41 ms)Preparing flush snapshotting stores in 1588230740 at 1732277608123Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732277608126 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732277608126Flushing 1588230740/info: creating writer at 1732277608128 (+2 ms)Flushing 1588230740/info: appending metadata at 1732277608181 (+53 ms)Flushing 1588230740/info: closing flushed file at 1732277608183 (+2 ms)Flushing 1588230740/ns: creating writer at 1732277608250 (+67 ms)Flushing 1588230740/ns: appending metadata at 1732277608265 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732277608265Flushing 1588230740/table: creating writer at 1732277608282 (+17 ms)Flushing 1588230740/table: appending metadata at 1732277608297 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732277608297Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@171e010c: reopening flushed file at 1732277608321 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@594752b3: reopening flushed file at 1732277608334 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3872812d: reopening flushed file at 1732277608345 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 237ms, sequenceid=60, compaction requested=false; wal=null at 1732277608363 (+18 ms)Cleaning up temporary data from old regions at 1732277608370 (+7 ms)Running coprocessor post-open hooks at 1732277608378 (+8 ms)Region opened successfully at 1732277608378 2024-11-22T12:13:28,380 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1732277608051 2024-11-22T12:13:28,382 DEBUG [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T12:13:28,382 INFO [RS_OPEN_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T12:13:28,383 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:28,384 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:28,400 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T12:13:28,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:28,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:28,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:28,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:28,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:28,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:28,433 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:28,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-22T12:13:28,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 in 537 msec 2024-11-22T12:13:28,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-22T12:13:28,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 758 msec 2024-11-22T12:13:28,440 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 46e18b47f3d3,37263,1732277602008 had 9 regions 2024-11-22T12:13:28,441 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true, isMeta: false 2024-11-22T12:13:28,444 INFO [PEWorker-1 {}] master.SplitWALManager(105): 46e18b47f3d3,37263,1732277602008 WAL count=1, meta=false 2024-11-22T12:13:28,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705}] 2024-11-22T12:13:28,446 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:28,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:28,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-22T12:13:28,622 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705, size=2.2 K (2237bytes) 2024-11-22T12:13:28,622 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705 2024-11-22T12:13:28,623 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705 after 1ms 2024-11-22T12:13:28,626 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705 took 4ms 2024-11-22T12:13:28,630 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d5254a456e6265d346df2dc9fbb1afe4: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-22T12:13:28,632 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4c61ef28412030a5de1748e0494b5d77: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-22T12:13:28,634 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d1e0cdf60c0d514100f271449411b4ed: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-22T12:13:28,636 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7cf86cd087fe778d3cb9ffcdb4f32fdf: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-22T12:13:28,638 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4f3a8ed48d3b002e0140f5d1f4422c45: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-22T12:13:28,640 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 88ee6f91206e330acd4027ac1712c4b5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-22T12:13:28,642 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c3b4530e36a471e0bfd4c64c45ec26d5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-22T12:13:28,644 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 5acff54db0b679ccee30c4cfa789fe5b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-22T12:13:28,644 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705 so closing down 2024-11-22T12:13:28,645 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-22T12:13:28,645 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-22T12:13:28,645 INFO [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-22T12:13:28,645 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705, journal: Splitting hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705, size=2.2 K (2237bytes) at 1732277608622Finishing writing output for hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705 so closing down at 1732277608644 (+22 ms)3 split writer threads finished at 1732277608645 (+1 ms)Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732277608645 2024-11-22T12:13:28,645 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705 2024-11-22T12:13:28,645 DEBUG [RS_LOG_REPLAY_OPS-regionserver/46e18b47f3d3:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-22T12:13:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35109 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-22T12:13:28,650 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting/46e18b47f3d3%2C37263%2C1732277602008.1732277603705 to hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs 2024-11-22T12:13:28,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-22T12:13:28,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 in 203 msec 2024-11-22T12:13:28,654 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:28,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-22T12:13:28,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 in 210 msec 2024-11-22T12:13:28,658 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/WALs/46e18b47f3d3,37263,1732277602008-splitting dir is empty, no logs to split. 2024-11-22T12:13:28,658 INFO [PEWorker-1 {}] master.SplitWALManager(105): 46e18b47f3d3,37263,1732277602008 WAL count=0, meta=false 2024-11-22T12:13:28,658 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 46e18b47f3d3,37263,1732277602008 WAL splitting is done? wals=0, meta=false 2024-11-22T12:13:28,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN}] 2024-11-22T12:13:28,663 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:28,664 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T12:13:28,665 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:28,665 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T12:13:28,666 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:28,667 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T12:13:28,668 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:28,669 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T12:13:28,670 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:28,671 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T12:13:28,672 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:28,673 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T12:13:28,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:28,675 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T12:13:28,677 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:28,678 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T12:13:28,815 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(204): Hosts are {46e18b47f3d3=0} racks are {/default-rack=0} 2024-11-22T12:13:28,815 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T12:13:28,815 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T12:13:28,815 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T12:13:28,815 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T12:13:28,815 INFO [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T12:13:28,815 INFO [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T12:13:28,815 DEBUG [46e18b47f3d3:35109 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T12:13:28,816 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:28,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:28,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:28,818 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, regionLocation=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:28,817 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 46e18b47f3d3:37263 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T12:13:28,818 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:28,819 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:28,819 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:28,820 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, regionLocation=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:28,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1, error=java.net.ConnectException: Call to address=46e18b47f3d3:37263 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-22T12:13:28,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 from cache 2024-11-22T12:13:28,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=46e18b47f3d3:37263 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 46e18b47f3d3:37263 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,823 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 46e18b47f3d3:37263 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T12:13:28,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=46e18b47f3d3:37263 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=46e18b47f3d3:37263 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=46e18b47f3d3:37263 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=46e18b47f3d3:37263 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=46e18b47f3d3:37263 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,37263,1732277602008, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=46e18b47f3d3:37263 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 46e18b47f3d3:37263 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 46e18b47f3d3/172.17.0.2:37263 2024-11-22T12:13:28,924 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T12:13:28,924 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=46e18b47f3d3,38629,1732277601859, seqNum=-1] 2024-11-22T12:13:28,924 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T12:13:28,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48199, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T12:13:28,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN because future has completed 2024-11-22T12:13:28,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:28,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN because future has completed 2024-11-22T12:13:28,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:28,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN because future has completed 2024-11-22T12:13:28,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:28,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN because future has completed 2024-11-22T12:13:28,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN because future has completed 2024-11-22T12:13:28,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN because future has completed 2024-11-22T12:13:28,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN because future has completed 2024-11-22T12:13:28,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:28,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN because future has completed 2024-11-22T12:13:28,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:28,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:28,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,40513,1732277601956}] 2024-11-22T12:13:28,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cf86cd087fe778d3cb9ffcdb4f32fdf, server=46e18b47f3d3,38629,1732277601859}] 2024-11-22T12:13:29,085 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:29,086 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 88ee6f91206e330acd4027ac1712c4b5, NAME => 'test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-22T12:13:29,086 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,086 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:29,087 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,087 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,088 INFO [StoreOpener-88ee6f91206e330acd4027ac1712c4b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,090 INFO [StoreOpener-88ee6f91206e330acd4027ac1712c4b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88ee6f91206e330acd4027ac1712c4b5 columnFamilyName family 2024-11-22T12:13:29,090 DEBUG [StoreOpener-88ee6f91206e330acd4027ac1712c4b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:29,090 INFO [StoreOpener-88ee6f91206e330acd4027ac1712c4b5-1 {}] regionserver.HStore(327): Store=88ee6f91206e330acd4027ac1712c4b5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:29,090 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,091 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,093 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,094 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,094 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,096 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,097 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 88ee6f91206e330acd4027ac1712c4b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58928570, jitterRate=-0.12189587950706482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:29,097 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:29,097 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 88ee6f91206e330acd4027ac1712c4b5: Running coprocessor pre-open hook at 1732277609087Writing region info on filesystem at 1732277609087Initializing all the Stores at 1732277609088 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277609088Cleaning up temporary data from old regions at 1732277609094 (+6 ms)Running coprocessor post-open hooks at 1732277609097 (+3 ms)Region opened successfully at 1732277609097 2024-11-22T12:13:29,098 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5., pid=72, masterSystemTime=1732277609081 2024-11-22T12:13:29,098 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c61ef28412030a5de1748e0494b5d77, NAME => 'test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-22T12:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,101 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:29,101 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:29,101 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:29,101 INFO [StoreOpener-4c61ef28412030a5de1748e0494b5d77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,102 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 5acff54db0b679ccee30c4cfa789fe5b, NAME => 'test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-22T12:13:29,102 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,102 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:29,102 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,102 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,103 INFO [StoreOpener-4c61ef28412030a5de1748e0494b5d77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c61ef28412030a5de1748e0494b5d77 columnFamilyName family 2024-11-22T12:13:29,103 DEBUG [StoreOpener-4c61ef28412030a5de1748e0494b5d77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:29,104 INFO [StoreOpener-4c61ef28412030a5de1748e0494b5d77-1 {}] regionserver.HStore(327): Store=4c61ef28412030a5de1748e0494b5d77/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:29,104 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,105 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-22T12:13:29,105 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,105 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,105 INFO [StoreOpener-5acff54db0b679ccee30c4cfa789fe5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,105 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,106 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,107 INFO [StoreOpener-5acff54db0b679ccee30c4cfa789fe5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5acff54db0b679ccee30c4cfa789fe5b columnFamilyName family 2024-11-22T12:13:29,107 DEBUG [StoreOpener-5acff54db0b679ccee30c4cfa789fe5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:29,107 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,108 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,108 INFO [StoreOpener-5acff54db0b679ccee30c4cfa789fe5b-1 {}] regionserver.HStore(327): Store=5acff54db0b679ccee30c4cfa789fe5b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:29,108 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,109 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,110 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,110 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,110 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,111 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,111 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 4c61ef28412030a5de1748e0494b5d77; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63204786, jitterRate=-0.05817529559135437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:29,111 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:29,111 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 4c61ef28412030a5de1748e0494b5d77: Running coprocessor pre-open hook at 1732277609099Writing region info on filesystem at 1732277609099Initializing all the Stores at 1732277609101 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277609101Cleaning up temporary data from old regions at 1732277609108 (+7 ms)Running coprocessor post-open hooks at 1732277609111 (+3 ms)Region opened successfully at 1732277609111 2024-11-22T12:13:29,112 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77., pid=77, masterSystemTime=1732277609093 2024-11-22T12:13:29,114 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,115 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:29,115 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:29,115 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:29,115 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f3a8ed48d3b002e0140f5d1f4422c45, NAME => 'test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-22T12:13:29,116 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 5acff54db0b679ccee30c4cfa789fe5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66888878, jitterRate=-0.0032780468463897705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:29,116 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,116 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:29,116 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:29,116 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 5acff54db0b679ccee30c4cfa789fe5b: Running coprocessor pre-open hook at 1732277609102Writing region info on filesystem at 1732277609102Initializing all the Stores at 1732277609105 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277609105Cleaning up temporary data from old regions at 1732277609111 (+6 ms)Running coprocessor post-open hooks at 1732277609116 (+5 ms)Region opened successfully at 1732277609116 2024-11-22T12:13:29,116 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,116 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,117 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b., pid=73, masterSystemTime=1732277609081 2024-11-22T12:13:29,116 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-22T12:13:29,118 INFO [StoreOpener-4f3a8ed48d3b002e0140f5d1f4422c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,118 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,118 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cf86cd087fe778d3cb9ffcdb4f32fdf, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,119 INFO [StoreOpener-4f3a8ed48d3b002e0140f5d1f4422c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f3a8ed48d3b002e0140f5d1f4422c45 columnFamilyName family 2024-11-22T12:13:29,119 DEBUG [StoreOpener-4f3a8ed48d3b002e0140f5d1f4422c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:29,119 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:29,120 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:29,120 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:29,120 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 7cf86cd087fe778d3cb9ffcdb4f32fdf, NAME => 'test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-22T12:13:29,120 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,120 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:29,121 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,121 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,121 INFO [StoreOpener-4f3a8ed48d3b002e0140f5d1f4422c45-1 {}] regionserver.HStore(327): Store=4f3a8ed48d3b002e0140f5d1f4422c45/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:29,122 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,123 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,123 INFO [StoreOpener-7cf86cd087fe778d3cb9ffcdb4f32fdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,124 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,124 INFO [StoreOpener-7cf86cd087fe778d3cb9ffcdb4f32fdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cf86cd087fe778d3cb9ffcdb4f32fdf columnFamilyName family 2024-11-22T12:13:29,125 DEBUG [StoreOpener-7cf86cd087fe778d3cb9ffcdb4f32fdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:29,125 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,125 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,125 INFO [StoreOpener-7cf86cd087fe778d3cb9ffcdb4f32fdf-1 {}] regionserver.HStore(327): Store=7cf86cd087fe778d3cb9ffcdb4f32fdf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:29,125 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,126 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cf86cd087fe778d3cb9ffcdb4f32fdf, server=46e18b47f3d3,38629,1732277601859 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,127 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,127 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,128 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 4f3a8ed48d3b002e0140f5d1f4422c45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70619867, jitterRate=0.0523180216550827}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:29,128 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,128 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:29,128 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 4f3a8ed48d3b002e0140f5d1f4422c45: Running coprocessor pre-open hook at 1732277609116Writing region info on filesystem at 1732277609116Initializing all the Stores at 1732277609117 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277609117Cleaning up temporary data from old regions at 1732277609125 (+8 ms)Running coprocessor post-open hooks at 1732277609128 (+3 ms)Region opened successfully at 1732277609128 2024-11-22T12:13:29,128 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,128 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,129 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45., pid=78, masterSystemTime=1732277609093 2024-11-22T12:13:29,131 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,132 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:29,132 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:29,132 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:29,132 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 7cf86cd087fe778d3cb9ffcdb4f32fdf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63575326, jitterRate=-0.05265381932258606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:29,132 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => d1e0cdf60c0d514100f271449411b4ed, NAME => 'test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-22T12:13:29,132 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:29,132 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 7cf86cd087fe778d3cb9ffcdb4f32fdf: Running coprocessor pre-open hook at 1732277609121Writing region info on filesystem at 1732277609121Initializing all the Stores at 1732277609123 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277609123Cleaning up temporary data from old regions at 1732277609128 (+5 ms)Running coprocessor post-open hooks at 1732277609132 (+4 ms)Region opened successfully at 1732277609132 2024-11-22T12:13:29,133 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,133 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:29,133 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,133 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,133 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf., pid=79, masterSystemTime=1732277609081 2024-11-22T12:13:29,134 INFO [StoreOpener-d1e0cdf60c0d514100f271449411b4ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,136 INFO [StoreOpener-d1e0cdf60c0d514100f271449411b4ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1e0cdf60c0d514100f271449411b4ed columnFamilyName family 2024-11-22T12:13:29,136 DEBUG [StoreOpener-d1e0cdf60c0d514100f271449411b4ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:29,136 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:29,136 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:29,136 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. 2024-11-22T12:13:29,136 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => d5254a456e6265d346df2dc9fbb1afe4, NAME => 'test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-22T12:13:29,136 INFO [StoreOpener-d1e0cdf60c0d514100f271449411b4ed-1 {}] regionserver.HStore(327): Store=d1e0cdf60c0d514100f271449411b4ed/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:29,136 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,136 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,137 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:29,137 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,137 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,137 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,138 INFO [StoreOpener-d5254a456e6265d346df2dc9fbb1afe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,139 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,139 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,139 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,139 INFO [StoreOpener-d5254a456e6265d346df2dc9fbb1afe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5254a456e6265d346df2dc9fbb1afe4 columnFamilyName family 2024-11-22T12:13:29,139 DEBUG [StoreOpener-d5254a456e6265d346df2dc9fbb1afe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:29,140 INFO [StoreOpener-d5254a456e6265d346df2dc9fbb1afe4-1 {}] regionserver.HStore(327): Store=d5254a456e6265d346df2dc9fbb1afe4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:29,140 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,141 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,141 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,142 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,142 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened d1e0cdf60c0d514100f271449411b4ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64863779, jitterRate=-0.033454373478889465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:29,142 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:29,142 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for d1e0cdf60c0d514100f271449411b4ed: Running coprocessor pre-open hook at 1732277609133Writing region info on filesystem at 1732277609133Initializing all the Stores at 1732277609134 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277609134Cleaning up temporary data from old regions at 1732277609139 (+5 ms)Running coprocessor post-open hooks at 1732277609142 (+3 ms)Region opened successfully at 1732277609142 2024-11-22T12:13:29,142 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,142 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,143 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed., pid=76, masterSystemTime=1732277609093 2024-11-22T12:13:29,146 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:29,146 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:29,146 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. 2024-11-22T12:13:29,146 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => c3b4530e36a471e0bfd4c64c45ec26d5, NAME => 'test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-22T12:13:29,146 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,146 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T12:13:29,146 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,147 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,149 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,150 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,150 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,150 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,152 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,152 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,152 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,153 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened d5254a456e6265d346df2dc9fbb1afe4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73986659, jitterRate=0.10248713195323944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:29,154 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d5254a456e6265d346df2dc9fbb1afe4 2024-11-22T12:13:29,154 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for d5254a456e6265d346df2dc9fbb1afe4: Running coprocessor pre-open hook at 1732277609137Writing region info on filesystem at 1732277609137Initializing all the Stores at 1732277609138 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277609138Cleaning up temporary data from old regions at 1732277609143 (+5 ms)Running coprocessor post-open hooks at 1732277609154 (+11 ms)Region opened successfully at 1732277609154 2024-11-22T12:13:29,155 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4., pid=74, masterSystemTime=1732277609081 2024-11-22T12:13:29,156 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,157 INFO [StoreOpener-c3b4530e36a471e0bfd4c64c45ec26d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,158 INFO [StoreOpener-c3b4530e36a471e0bfd4c64c45ec26d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3b4530e36a471e0bfd4c64c45ec26d5 columnFamilyName family 2024-11-22T12:13:29,158 DEBUG [StoreOpener-c3b4530e36a471e0bfd4c64c45ec26d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T12:13:29,159 INFO [StoreOpener-c3b4530e36a471e0bfd4c64c45ec26d5-1 {}] regionserver.HStore(327): Store=c3b4530e36a471e0bfd4c64c45ec26d5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T12:13:29,158 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "46e18b47f3d3" port: 38629 start_code: 1732277601859 } transition { transition_code: OPENED region_info { region_id: 1732277604754 table_name { namespace: "default" qualifier: "test" } start_key: "iii" end_key: "jjj" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 74 initiating_master_active_time: 1732277602108 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-22T12:13:29,159 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,160 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,161 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,164 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,165 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened c3b4530e36a471e0bfd4c64c45ec26d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67375324, jitterRate=0.0039705634117126465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T12:13:29,165 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3b4530e36a471e0bfd4c64c45ec26d5 2024-11-22T12:13:29,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,166 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for c3b4530e36a471e0bfd4c64c45ec26d5: Running coprocessor pre-open hook at 1732277609147Writing region info on filesystem at 1732277609147Initializing all the Stores at 1732277609151 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732277609151Cleaning up temporary data from old regions at 1732277609162 (+11 ms)Running coprocessor post-open hooks at 1732277609165 (+3 ms)Region opened successfully at 1732277609165 2024-11-22T12:13:29,166 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5., pid=75, masterSystemTime=1732277609093 2024-11-22T12:13:29,167 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,168 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "46e18b47f3d3" port: 40513 start_code: 1732277601956 } transition { transition_code: OPENED region_info { region_id: 1732277604754 table_name { namespace: "default" qualifier: "test" } start_key: "lll" end_key: "mmm" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 75 initiating_master_active_time: 1732277602108 }; retry (#0) after 1006ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-22T12:13:29,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cf86cd087fe778d3cb9ffcdb4f32fdf, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,172 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,173 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,173 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,173 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,173 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,173 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,173 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,173 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,173 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,173 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,174 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,174 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,174 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,174 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,174 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,174 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,174 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,174 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cf86cd087fe778d3cb9ffcdb4f32fdf, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,174 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 21 msec 2024-11-22T12:13:29,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,177 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,177 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,224 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,224 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,225 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN to state=OFFLINE, location=null, table=test, region=5acff54db0b679ccee30c4cfa789fe5b to restore RIT 2024-11-22T12:13:29,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639226 2024-11-22T12:13:29,226 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,226 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,228 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,230 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,230 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,230 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,40513,1732277601956 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,262 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,262 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669262 2024-11-22T12:13:29,262 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729262 2024-11-22T12:13:29,262 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 35 msec 2024-11-22T12:13:29,262 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,262 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,262 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,262 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,263 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,263 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,263 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,263 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,264 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T12:13:29,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,272 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,279 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,281 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,281 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,281 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,281 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,281 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,281 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,281 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f3a8ed48d3b002e0140f5d1f4422c45, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,281 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-22T12:13:29,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,283 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,283 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,297 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,297 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,297 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,297 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,297 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,297 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,297 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,297 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN to state=OFFLINE, location=null, table=test, region=5acff54db0b679ccee30c4cfa789fe5b to restore RIT 2024-11-22T12:13:29,297 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639297 2024-11-22T12:13:29,297 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,297 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,299 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,300 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,300 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,301 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,40513,1732277601956 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,315 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,316 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669316 2024-11-22T12:13:29,316 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729316 2024-11-22T12:13:29,316 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-22T12:13:29,316 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,316 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,316 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,316 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,316 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,316 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,316 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,317 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,318 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T12:13:29,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,328 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,329 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,332 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,332 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,332 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,333 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,333 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,333 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,333 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,333 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c61ef28412030a5de1748e0494b5d77, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,333 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-22T12:13:29,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,335 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,335 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,349 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,349 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,349 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,350 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,350 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,350 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,350 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,350 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,350 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,350 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,350 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN to state=OFFLINE, location=null, table=test, region=5acff54db0b679ccee30c4cfa789fe5b to restore RIT 2024-11-22T12:13:29,350 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639350 2024-11-22T12:13:29,351 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,351 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,355 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,357 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,357 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,357 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,364 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,364 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669364 2024-11-22T12:13:29,364 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729364 2024-11-22T12:13:29,364 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-22T12:13:29,364 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,364 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,364 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,364 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,364 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,364 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,364 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,365 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,366 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,375 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,375 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,375 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,375 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,376 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,376 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,376 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,376 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,376 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1e0cdf60c0d514100f271449411b4ed, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,376 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-22T12:13:29,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,377 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,377 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,392 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,392 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,392 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,392 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN to state=OFFLINE, location=null, table=test, region=5acff54db0b679ccee30c4cfa789fe5b to restore RIT 2024-11-22T12:13:29,392 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639392 2024-11-22T12:13:29,392 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,392 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,394 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,395 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,395 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,395 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,404 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,404 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669404 2024-11-22T12:13:29,404 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729404 2024-11-22T12:13:29,404 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-22T12:13:29,404 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,404 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,404 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,404 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,404 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,405 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,405 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,413 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,413 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,414 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,414 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,414 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,414 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b4530e36a471e0bfd4c64c45ec26d5, server=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,414 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-22T12:13:29,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,415 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,415 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN to state=OFFLINE, location=null, table=test, region=5acff54db0b679ccee30c4cfa789fe5b to restore RIT 2024-11-22T12:13:29,423 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639423 2024-11-22T12:13:29,423 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,423 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,425 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,426 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,426 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,426 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,433 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,433 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669433 2024-11-22T12:13:29,433 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729433 2024-11-22T12:13:29,433 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-22T12:13:29,433 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,433 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,433 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,434 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,434 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,434 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,434 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,434 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,435 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,438 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,439 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,441 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,441 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,441 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,441 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,442 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,442 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,442 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,442 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,442 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,442 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,442 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,442 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5254a456e6265d346df2dc9fbb1afe4, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,442 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T12:13:29,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,443 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,443 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,454 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,454 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,454 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,454 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,454 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,455 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,455 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,455 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,455 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,455 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,455 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN to state=OFFLINE, location=null, table=test, region=5acff54db0b679ccee30c4cfa789fe5b to restore RIT 2024-11-22T12:13:29,455 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639455 2024-11-22T12:13:29,455 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,455 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,456 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,457 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,457 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,457 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,464 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669464 2024-11-22T12:13:29,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729464 2024-11-22T12:13:29,464 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-22T12:13:29,464 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,464 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,464 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,464 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,465 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,465 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,465 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,465 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,472 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,473 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,475 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,475 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,475 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,475 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,475 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,475 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,475 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,475 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,475 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acff54db0b679ccee30c4cfa789fe5b, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,475 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-22T12:13:29,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,476 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,476 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN to state=OFFLINE, location=null, table=test, region=5acff54db0b679ccee30c4cfa789fe5b to restore RIT 2024-11-22T12:13:29,487 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639487 2024-11-22T12:13:29,487 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,487 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,488 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,489 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,489 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,489 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,495 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,495 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669495 2024-11-22T12:13:29,495 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729495 2024-11-22T12:13:29,495 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-22T12:13:29,495 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,495 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,495 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,495 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,496 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,496 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,496 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,496 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,497 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,498 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,498 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,498 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,502 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,503 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T12:13:29,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,504 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,504 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,517 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,518 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,518 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,518 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,518 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,518 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,518 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,518 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN to state=OFFLINE, location=null, table=test, region=5acff54db0b679ccee30c4cfa789fe5b to restore RIT 2024-11-22T12:13:29,519 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639519 2024-11-22T12:13:29,519 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,519 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,520 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,529 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,529 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,529 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669529 2024-11-22T12:13:29,529 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729529 2024-11-22T12:13:29,529 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-22T12:13:29,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,529 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,529 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,529 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,529 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,529 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,529 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,530 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,530 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,531 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,534 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,536 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,536 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN 2024-11-22T12:13:29,538 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-22T12:13:29,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,539 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,539 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,549 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,549 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,549 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,550 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,550 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,550 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,550 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,550 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,550 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,550 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,550 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5acff54db0b679ccee30c4cfa789fe5b, ASSIGN to state=OFFLINE, location=null, table=test, region=5acff54db0b679ccee30c4cfa789fe5b to restore RIT 2024-11-22T12:13:29,553 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639553 2024-11-22T12:13:29,553 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,553 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,554 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,555 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,555 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,555 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,561 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,561 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669561 2024-11-22T12:13:29,561 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729561 2024-11-22T12:13:29,561 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-22T12:13:29,562 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,562 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,562 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,562 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,562 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,562 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,562 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,562 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,563 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,567 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,568 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN 2024-11-22T12:13:29,568 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T12:13:29,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,569 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,569 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,582 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,582 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ee6f91206e330acd4027ac1712c4b5, ASSIGN to state=OFFLINE, location=null, table=test, region=88ee6f91206e330acd4027ac1712c4b5 to restore RIT 2024-11-22T12:13:29,582 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639582 2024-11-22T12:13:29,582 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,582 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,584 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,585 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,585 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,585 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,589 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669589 2024-11-22T12:13:29,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729589 2024-11-22T12:13:29,590 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-22T12:13:29,590 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,590 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,590 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,590 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,590 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,590 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,590 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,590 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN 2024-11-22T12:13:29,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T12:13:29,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,596 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,597 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,602 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,602 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,602 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,602 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,602 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,602 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4c61ef28412030a5de1748e0494b5d77, ASSIGN to state=OFFLINE, location=null, table=test, region=4c61ef28412030a5de1748e0494b5d77 to restore RIT 2024-11-22T12:13:29,603 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639602 2024-11-22T12:13:29,603 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,603 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,604 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,605 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,605 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,605 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,610 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,610 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669610 2024-11-22T12:13:29,610 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729610 2024-11-22T12:13:29,610 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-22T12:13:29,610 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,610 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,611 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,611 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,611 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,611 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,611 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,612 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,617 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,617 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,617 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,617 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,617 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN 2024-11-22T12:13:29,617 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T12:13:29,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,618 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,618 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,623 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,623 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,623 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,623 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,623 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf, ASSIGN to state=OFFLINE, location=null, table=test, region=7cf86cd087fe778d3cb9ffcdb4f32fdf to restore RIT 2024-11-22T12:13:29,624 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639623 2024-11-22T12:13:29,624 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,624 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,625 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,629 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,629 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,629 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,632 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,632 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669632 2024-11-22T12:13:29,632 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729632 2024-11-22T12:13:29,632 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-22T12:13:29,632 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,632 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,632 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,633 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,633 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,633 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,633 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,633 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,634 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,635 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,636 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,638 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,638 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,638 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,639 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,639 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,639 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN 2024-11-22T12:13:29,639 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T12:13:29,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,639 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,640 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,640 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,640 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,652 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,652 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,652 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,653 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,653 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,653 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,653 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d5254a456e6265d346df2dc9fbb1afe4, ASSIGN to state=OFFLINE, location=null, table=test, region=d5254a456e6265d346df2dc9fbb1afe4 to restore RIT 2024-11-22T12:13:29,654 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639654 2024-11-22T12:13:29,654 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,654 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,655 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,656 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,656 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,656 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,661 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,661 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669661 2024-11-22T12:13:29,661 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729661 2024-11-22T12:13:29,661 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-22T12:13:29,661 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,661 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,661 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,662 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,662 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,662 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,662 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,663 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,667 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,667 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,667 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,667 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN 2024-11-22T12:13:29,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T12:13:29,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,668 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,668 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,675 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,675 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,675 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1e0cdf60c0d514100f271449411b4ed, ASSIGN to state=OFFLINE, location=null, table=test, region=d1e0cdf60c0d514100f271449411b4ed to restore RIT 2024-11-22T12:13:29,676 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639676 2024-11-22T12:13:29,676 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,676 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,678 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,679 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,679 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,679 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,686 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669686 2024-11-22T12:13:29,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729686 2024-11-22T12:13:29,686 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-22T12:13:29,686 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,686 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,686 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,686 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,686 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,686 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,686 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,687 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,688 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,690 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,691 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,693 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,693 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,693 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN 2024-11-22T12:13:29,693 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T12:13:29,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,695 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,695 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,738 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,738 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,738 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,739 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,739 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45, ASSIGN to state=OFFLINE, location=null, table=test, region=4f3a8ed48d3b002e0140f5d1f4422c45 to restore RIT 2024-11-22T12:13:29,740 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639739 2024-11-22T12:13:29,740 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,740 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,741 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,742 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,743 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,743 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,749 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,749 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669749 2024-11-22T12:13:29,749 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729749 2024-11-22T12:13:29,749 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-22T12:13:29,749 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,749 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,749 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,749 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,750 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,750 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,750 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,750 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,751 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,754 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,754 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,754 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,755 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,756 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,756 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN 2024-11-22T12:13:29,756 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T12:13:29,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,757 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,758 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,780 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,780 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,780 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,781 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5, ASSIGN to state=OFFLINE, location=null, table=test, region=c3b4530e36a471e0bfd4c64c45ec26d5 to restore RIT 2024-11-22T12:13:29,781 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639781 2024-11-22T12:13:29,782 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,782 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,783 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,784 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,784 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,785 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,791 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,791 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669791 2024-11-22T12:13:29,791 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729791 2024-11-22T12:13:29,791 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-22T12:13:29,791 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,791 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,791 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,791 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,791 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,791 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,791 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,792 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,796 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-22T12:13:29,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,797 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,797 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,802 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639802 2024-11-22T12:13:29,802 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,802 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,803 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,805 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,805 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,805 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,809 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669809 2024-11-22T12:13:29,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729809 2024-11-22T12:13:29,809 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-22T12:13:29,809 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,809 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,809 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,809 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,810 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,810 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,810 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,812 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,812 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,812 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,812 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,812 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,812 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,813 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.1732277603705, worker=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,814 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-22T12:13:29,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,814 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,814 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,822 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,822 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,822 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,823 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639823 2024-11-22T12:13:29,823 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,823 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,825 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,826 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,826 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,826 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,831 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,831 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669831 2024-11-22T12:13:29,831 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729831 2024-11-22T12:13:29,831 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-22T12:13:29,831 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:29,831 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,831 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:29,831 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:29,831 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:29,831 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:29,831 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:29,831 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:29,832 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:29,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:29,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:29,834 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T12:13:29,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:29,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:29,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-22T12:13:29,834 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:29,834 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:29,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:29,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:29,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:29,835 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:29,835 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,835 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,836 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,836 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46e18b47f3d3,38629,1732277601859 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:29,836 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-22T12:13:29,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:29,837 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:29,837 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:29,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:29,843 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,843 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,843 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:29,846 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277639846 2024-11-22T12:13:29,846 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:29,846 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,848 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:29,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:29,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:29,855 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:29,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277669855 2024-11-22T12:13:29,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277729855 2024-11-22T12:13:29,855 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-22T12:13:29,860 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:29,860 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:29,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:29,860 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:29,891 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-22T12:13:30,105 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:30,106 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:30,106 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:30,106 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:30,106 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:30,106 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:30,106 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:30,107 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:30,107 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:30,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:30,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:30,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:30,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T12:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:30,110 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:30,110 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T12:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:30,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-22T12:13:30,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:30,112 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:30,112 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:30,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,118 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277640118 2024-11-22T12:13:30,118 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:30,118 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,120 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:30,121 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:30,121 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:30,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:30,121 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,126 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:30,126 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277670126 2024-11-22T12:13:30,126 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277730126 2024-11-22T12:13:30,126 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-22T12:13:30,126 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:30,126 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:30,126 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:30,126 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:30,126 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:30,126 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:30,126 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:30,127 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:30,127 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:30,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:30,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:30,129 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-22T12:13:30,129 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:30,129 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:30,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:30,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:30,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:30,130 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:30,130 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:30,130 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:30,130 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-22T12:13:30,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:30,131 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:30,131 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:30,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,139 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277640139 2024-11-22T12:13:30,139 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:30,139 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,144 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:30,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:30,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,146 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:30,146 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277670146 2024-11-22T12:13:30,146 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277730146 2024-11-22T12:13:30,146 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-22T12:13:30,147 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:30,147 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:30,147 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:30,147 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:30,147 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:30,147 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:30,147 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:30,147 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:30,148 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:30,149 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:30,149 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-22T12:13:30,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:30,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:30,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:30,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:30,151 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:30,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 46e18b47f3d3%2C37263%2C1732277602008.meta.1732277604276.meta, worker=46e18b47f3d3,40513,1732277601956 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T12:13:30,151 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-22T12:13:30,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:30,151 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:30,152 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:30,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,160 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277640160 2024-11-22T12:13:30,160 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:30,160 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,161 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:30,162 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-22T12:13:30,162 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:30,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:30,162 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,165 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,168 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:30,168 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277670168 2024-11-22T12:13:30,168 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277730168 2024-11-22T12:13:30,168 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-22T12:13:30,168 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T12:13:30,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:30,168 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:30,168 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:30,168 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T12:13:30,168 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T12:13:30,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T12:13:30,169 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.CallRunner(138): callId: 85 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 126 connection: 172.17.0.2:42147 deadline: 1732277620165, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-11-22T12:13:30,169 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T12:13:30,169 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,169 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,170 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T12:13:30,169 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "46e18b47f3d3" port: 38629 start_code: 1732277601859 } transition { transition_code: OPENED region_info { region_id: 1732277604754 table_name { namespace: "default" qualifier: "test" } start_key: "iii" end_key: "jjj" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 74 initiating_master_active_time: 1732277602108 }; retry (#1) after 2013ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-22T12:13:30,171 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-22T12:13:30,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T12:13:30,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-22T12:13:30,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true 2024-11-22T12:13:30,172 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-22T12:13:30,173 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-22T12:13:30,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=61 2024-11-22T12:13:30,173 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=46e18b47f3d3,38629,1732277601859, table=hbase:meta, region=1588230740 2024-11-22T12:13:30,174 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46e18b47f3d3,38629,1732277601859, state=OPEN 2024-11-22T12:13:30,176 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,177 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "46e18b47f3d3" port: 40513 start_code: 1732277601956 } transition { transition_code: OPENED region_info { region_id: 1732277604754 table_name { namespace: "default" qualifier: "test" } start_key: "lll" end_key: "mmm" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 75 initiating_master_active_time: 1732277602108 }; retry (#1) after 2008ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-22T12:13:30,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T12:13:30,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T12:13:30,181 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732277640181 2024-11-22T12:13:30,181 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T12:13:30,181 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-22T12:13:30,182 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-22T12:13:30,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 88ee6f91206e330acd4027ac1712c4b5, server=46e18b47f3d3,38629,1732277601859:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 46e18b47f3d3,37263,1732277602008, splitWal=true, meta=true exec-time=2.9270 sec 2024-11-22T12:13:30,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ff71fd9f56108f32cc070df1485ba23, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=142956bd7cdd41ce32216563195a828d, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cf86cd087fe778d3cb9ffcdb4f32fdf, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=800c34543b39c820cefb84a25e72255c, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aef6551d501af493aaa4fd1077bf9c9, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1e0cdf60c0d514100f271449411b4ed, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4584ab1e18d121599a608cdb4be19ea, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33fb94d3dd5df2c85edbc81c792b792, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d5254a456e6265d346df2dc9fbb1afe4, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b05283d6644e96595068a6390bc0e1fe, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c48682b2ea1380365b4d638dd13d2a9, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3b4530e36a471e0bfd4c64c45ec26d5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb5b2e2a4461c2824dc11a5a423a6d19, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=794bd231e579bf5991f118c3359c87ab, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c19c84c4949f0cb43eed608a20910dc, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acff54db0b679ccee30c4cfa789fe5b, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88a8bf4fbe7648785154bd80df699f0b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dbb25f70e3828779de1b8447a41064c, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c61ef28412030a5de1748e0494b5d77, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca3907a8067af4c43fffaebaa942770, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f061dd4aec8e910fcda8dcf65d18b2b, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ee6f91206e330acd4027ac1712c4b5, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1716ab8b0523ebd9a28d11dac1fd8e6, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b745bd27cbcb9cc89fc04815d334ae24, regionState=OPEN, lastHost=46e18b47f3d3,40513,1732277601956, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f3a8ed48d3b002e0140f5d1f4422c45, regionState=OPENING, lastHost=46e18b47f3d3,37263,1732277602008, regionLocation=46e18b47f3d3,40513,1732277601956, openSeqNum=2 2024-11-22T12:13:30,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a3cd019b15b8a97ff8eeb1bbb051471, regionState=OPEN, lastHost=46e18b47f3d3,38629,1732277601859, regionLocation=46e18b47f3d3,38629,1732277601859, openSeqNum=2 2024-11-22T12:13:30,187 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T12:13:30,187 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732277670187 2024-11-22T12:13:30,187 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732277730187 2024-11-22T12:13:30,187 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-22T12:13:30,203 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=347 (was 380), OpenFileDescriptor=616 (was 638), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 328) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2586 (was 2663) 2024-11-22T12:13:30,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T12:13:30,203 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T12:13:30,203 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T12:13:30,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:30,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:30,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T12:13:30,204 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T12:13:30,204 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1816000828, stopped=false 2024-11-22T12:13:30,204 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=46e18b47f3d3,35109,1732277601092 2024-11-22T12:13:30,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T12:13:30,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T12:13:30,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T12:13:30,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:30,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:30,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:30,212 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:30,212 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T12:13:30,212 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T12:13:30,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:30,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T12:13:30,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T12:13:30,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T12:13:30,212 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '46e18b47f3d3,38629,1732277601859' ***** 2024-11-22T12:13:30,213 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T12:13:30,213 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '46e18b47f3d3,40513,1732277601956' ***** 2024-11-22T12:13:30,213 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T12:13:30,213 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T12:13:30,213 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T12:13:30,213 INFO [RS:0;46e18b47f3d3:38629 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T12:13:30,213 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T12:13:30,213 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T12:13:30,213 INFO [RS:0;46e18b47f3d3:38629 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T12:13:30,213 INFO [RS:1;46e18b47f3d3:40513 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T12:13:30,213 INFO [RS:1;46e18b47f3d3:40513 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T12:13:30,213 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for 794bd231e579bf5991f118c3359c87ab 2024-11-22T12:13:30,213 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for 5acff54db0b679ccee30c4cfa789fe5b 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for 7f061dd4aec8e910fcda8dcf65d18b2b 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for 6c48682b2ea1380365b4d638dd13d2a9 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for 6ff71fd9f56108f32cc070df1485ba23 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for 1c19c84c4949f0cb43eed608a20910dc 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for b05283d6644e96595068a6390bc0e1fe 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for 800c34543b39c820cefb84a25e72255c 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for 6dbb25f70e3828779de1b8447a41064c 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for 142956bd7cdd41ce32216563195a828d 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a3cd019b15b8a97ff8eeb1bbb051471 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for 4f3a8ed48d3b002e0140f5d1f4422c45 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for d1e0cdf60c0d514100f271449411b4ed 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for 7aef6551d501af493aaa4fd1077bf9c9 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for 4c61ef28412030a5de1748e0494b5d77 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for 88ee6f91206e330acd4027ac1712c4b5 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for b4584ab1e18d121599a608cdb4be19ea 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for b745bd27cbcb9cc89fc04815d334ae24 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for 7cf86cd087fe778d3cb9ffcdb4f32fdf 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(3091): Received CLOSE for 88a8bf4fbe7648785154bd80df699f0b 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for cca3907a8067af4c43fffaebaa942770 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for d1716ab8b0523ebd9a28d11dac1fd8e6 2024-11-22T12:13:30,214 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 794bd231e579bf5991f118c3359c87ab, disabling compactions & flushes 2024-11-22T12:13:30,214 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5acff54db0b679ccee30c4cfa789fe5b, disabling compactions & flushes 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(959): stopping server 46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(959): stopping server 46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T12:13:30,214 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:30,214 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:30,214 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:30,214 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:30,214 INFO [RS:1;46e18b47f3d3:40513 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;46e18b47f3d3:40513. 2024-11-22T12:13:30,214 INFO [RS:0;46e18b47f3d3:38629 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;46e18b47f3d3:38629. 2024-11-22T12:13:30,214 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. after waiting 0 ms 2024-11-22T12:13:30,214 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. after waiting 0 ms 2024-11-22T12:13:30,215 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:30,215 DEBUG [RS:1;46e18b47f3d3:40513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T12:13:30,215 DEBUG [RS:0;46e18b47f3d3:38629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T12:13:30,215 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:30,215 DEBUG [RS:1;46e18b47f3d3:40513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:30,215 DEBUG [RS:0;46e18b47f3d3:38629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:30,215 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-22T12:13:30,215 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T12:13:30,215 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T12:13:30,215 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T12:13:30,215 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T12:13:30,215 DEBUG [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(1351): Waiting on 142956bd7cdd41ce32216563195a828d, 1c19c84c4949f0cb43eed608a20910dc, 4c61ef28412030a5de1748e0494b5d77, 4f3a8ed48d3b002e0140f5d1f4422c45, 6c48682b2ea1380365b4d638dd13d2a9, 794bd231e579bf5991f118c3359c87ab, 7f061dd4aec8e910fcda8dcf65d18b2b, 800c34543b39c820cefb84a25e72255c, 88a8bf4fbe7648785154bd80df699f0b, b745bd27cbcb9cc89fc04815d334ae24, c3b4530e36a471e0bfd4c64c45ec26d5, d1e0cdf60c0d514100f271449411b4ed, e33fb94d3dd5df2c85edbc81c792b792 2024-11-22T12:13:30,215 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-22T12:13:30,216 DEBUG [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5acff54db0b679ccee30c4cfa789fe5b, 6dbb25f70e3828779de1b8447a41064c, 6ff71fd9f56108f32cc070df1485ba23, 7aef6551d501af493aaa4fd1077bf9c9, 7cf86cd087fe778d3cb9ffcdb4f32fdf, 88ee6f91206e330acd4027ac1712c4b5, 8a3cd019b15b8a97ff8eeb1bbb051471, b05283d6644e96595068a6390bc0e1fe, b4584ab1e18d121599a608cdb4be19ea, cca3907a8067af4c43fffaebaa942770, d1716ab8b0523ebd9a28d11dac1fd8e6, d5254a456e6265d346df2dc9fbb1afe4, eb5b2e2a4461c2824dc11a5a423a6d19 2024-11-22T12:13:30,216 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T12:13:30,216 INFO [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T12:13:30,216 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T12:13:30,216 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T12:13:30,216 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T12:13:30,216 INFO [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-22T12:13:30,220 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/5acff54db0b679ccee30c4cfa789fe5b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,220 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/794bd231e579bf5991f118c3359c87ab/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,222 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:30,222 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:30,222 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5acff54db0b679ccee30c4cfa789fe5b: Waiting for close lock at 1732277610214Running coprocessor pre-close hooks at 1732277610214Disabling compacts and flushes for region at 1732277610214Disabling writes for close at 1732277610215 (+1 ms)Writing region close event to WAL at 1732277610215Running coprocessor post-close hooks at 1732277610220 (+5 ms)Closed at 1732277610222 (+2 ms) 2024-11-22T12:13:30,222 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 794bd231e579bf5991f118c3359c87ab: Waiting for close lock at 1732277610214Running coprocessor pre-close hooks at 1732277610214Disabling compacts and flushes for region at 1732277610214Disabling writes for close at 1732277610215 (+1 ms)Writing region close event to WAL at 1732277610215Running coprocessor post-close hooks at 1732277610220 (+5 ms)Closed at 1732277610222 (+2 ms) 2024-11-22T12:13:30,222 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732277604754.5acff54db0b679ccee30c4cfa789fe5b. 2024-11-22T12:13:30,222 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732277604754.794bd231e579bf5991f118c3359c87ab. 2024-11-22T12:13:30,223 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6ff71fd9f56108f32cc070df1485ba23, disabling compactions & flushes 2024-11-22T12:13:30,223 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7f061dd4aec8e910fcda8dcf65d18b2b, disabling compactions & flushes 2024-11-22T12:13:30,223 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:30,223 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:30,223 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:30,223 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. after waiting 0 ms 2024-11-22T12:13:30,223 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:30,223 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:30,223 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. after waiting 0 ms 2024-11-22T12:13:30,223 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:30,229 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6ff71fd9f56108f32cc070df1485ba23/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,230 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:30,230 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6ff71fd9f56108f32cc070df1485ba23: Waiting for close lock at 1732277610222Running coprocessor pre-close hooks at 1732277610223 (+1 ms)Disabling compacts and flushes for region at 1732277610223Disabling writes for close at 1732277610223Writing region close event to WAL at 1732277610223Running coprocessor post-close hooks at 1732277610229 (+6 ms)Closed at 1732277610230 (+1 ms) 2024-11-22T12:13:30,230 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732277604754.6ff71fd9f56108f32cc070df1485ba23. 2024-11-22T12:13:30,230 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b05283d6644e96595068a6390bc0e1fe, disabling compactions & flushes 2024-11-22T12:13:30,230 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:30,230 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:30,230 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. after waiting 0 ms 2024-11-22T12:13:30,230 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:30,233 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7f061dd4aec8e910fcda8dcf65d18b2b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,234 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/info/570f02170023490181fcfa97e06a0c30 is 138, key is test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf./info:regioninfo/1732277608818/Put/seqid=0 2024-11-22T12:13:30,234 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:30,234 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7f061dd4aec8e910fcda8dcf65d18b2b: Waiting for close lock at 1732277610223Running coprocessor pre-close hooks at 1732277610223Disabling compacts and flushes for region at 1732277610223Disabling writes for close at 1732277610223Writing region close event to WAL at 1732277610223Running coprocessor post-close hooks at 1732277610234 (+11 ms)Closed at 1732277610234 2024-11-22T12:13:30,234 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732277604754.7f061dd4aec8e910fcda8dcf65d18b2b. 2024-11-22T12:13:30,234 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6c48682b2ea1380365b4d638dd13d2a9, disabling compactions & flushes 2024-11-22T12:13:30,235 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:30,235 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:30,235 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. after waiting 0 ms 2024-11-22T12:13:30,235 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:30,242 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b05283d6644e96595068a6390bc0e1fe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,242 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6c48682b2ea1380365b4d638dd13d2a9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,243 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b05283d6644e96595068a6390bc0e1fe: Waiting for close lock at 1732277610230Running coprocessor pre-close hooks at 1732277610230Disabling compacts and flushes for region at 1732277610230Disabling writes for close at 1732277610230Writing region close event to WAL at 1732277610234 (+4 ms)Running coprocessor post-close hooks at 1732277610243 (+9 ms)Closed at 1732277610243 2024-11-22T12:13:30,243 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6c48682b2ea1380365b4d638dd13d2a9: Waiting for close lock at 1732277610234Running coprocessor pre-close hooks at 1732277610234Disabling compacts and flushes for region at 1732277610234Disabling writes for close at 1732277610235 (+1 ms)Writing region close event to WAL at 1732277610235Running coprocessor post-close hooks at 1732277610243 (+8 ms)Closed at 1732277610243 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732277604754.b05283d6644e96595068a6390bc0e1fe. 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732277604754.6c48682b2ea1380365b4d638dd13d2a9. 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6dbb25f70e3828779de1b8447a41064c, disabling compactions & flushes 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1c19c84c4949f0cb43eed608a20910dc, disabling compactions & flushes 2024-11-22T12:13:30,243 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:30,243 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. after waiting 0 ms 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. after waiting 0 ms 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:30,243 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:30,252 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/1c19c84c4949f0cb43eed608a20910dc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,253 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:30,253 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1c19c84c4949f0cb43eed608a20910dc: Waiting for close lock at 1732277610243Running coprocessor pre-close hooks at 1732277610243Disabling compacts and flushes for region at 1732277610243Disabling writes for close at 1732277610243Writing region close event to WAL at 1732277610243Running coprocessor post-close hooks at 1732277610253 (+10 ms)Closed at 1732277610253 2024-11-22T12:13:30,253 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732277604754.1c19c84c4949f0cb43eed608a20910dc. 2024-11-22T12:13:30,253 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 800c34543b39c820cefb84a25e72255c, disabling compactions & flushes 2024-11-22T12:13:30,253 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:30,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741869_1045 (size=8195) 2024-11-22T12:13:30,253 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:30,253 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. after waiting 0 ms 2024-11-22T12:13:30,253 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:30,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741869_1045 (size=8195) 2024-11-22T12:13:30,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741869_1045 (size=8195) 2024-11-22T12:13:30,256 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/6dbb25f70e3828779de1b8447a41064c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,256 INFO [regionserver/46e18b47f3d3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T12:13:30,257 INFO [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/info/570f02170023490181fcfa97e06a0c30 2024-11-22T12:13:30,257 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:30,257 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6dbb25f70e3828779de1b8447a41064c: Waiting for close lock at 1732277610243Running coprocessor pre-close hooks at 1732277610243Disabling compacts and flushes for region at 1732277610243Disabling writes for close at 1732277610243Writing region close event to WAL at 1732277610251 (+8 ms)Running coprocessor post-close hooks at 1732277610257 (+6 ms)Closed at 1732277610257 2024-11-22T12:13:30,258 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732277604754.6dbb25f70e3828779de1b8447a41064c. 2024-11-22T12:13:30,258 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a3cd019b15b8a97ff8eeb1bbb051471, disabling compactions & flushes 2024-11-22T12:13:30,258 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:30,258 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:30,258 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. after waiting 0 ms 2024-11-22T12:13:30,258 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:30,259 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/800c34543b39c820cefb84a25e72255c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,260 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:30,260 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 800c34543b39c820cefb84a25e72255c: Waiting for close lock at 1732277610253Running coprocessor pre-close hooks at 1732277610253Disabling compacts and flushes for region at 1732277610253Disabling writes for close at 1732277610253Writing region close event to WAL at 1732277610254 (+1 ms)Running coprocessor post-close hooks at 1732277610260 (+6 ms)Closed at 1732277610260 2024-11-22T12:13:30,260 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732277604754.800c34543b39c820cefb84a25e72255c. 2024-11-22T12:13:30,260 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 142956bd7cdd41ce32216563195a828d, disabling compactions & flushes 2024-11-22T12:13:30,260 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:30,260 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:30,260 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. after waiting 0 ms 2024-11-22T12:13:30,260 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:30,265 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/8a3cd019b15b8a97ff8eeb1bbb051471/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,265 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/142956bd7cdd41ce32216563195a828d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,266 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:30,266 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a3cd019b15b8a97ff8eeb1bbb051471: Waiting for close lock at 1732277610258Running coprocessor pre-close hooks at 1732277610258Disabling compacts and flushes for region at 1732277610258Disabling writes for close at 1732277610258Writing region close event to WAL at 1732277610258Running coprocessor post-close hooks at 1732277610266 (+8 ms)Closed at 1732277610266 2024-11-22T12:13:30,266 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732277604754.8a3cd019b15b8a97ff8eeb1bbb051471. 2024-11-22T12:13:30,266 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb5b2e2a4461c2824dc11a5a423a6d19, disabling compactions & flushes 2024-11-22T12:13:30,266 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:30,266 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:30,266 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. after waiting 0 ms 2024-11-22T12:13:30,266 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:30,266 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:30,267 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 142956bd7cdd41ce32216563195a828d: Waiting for close lock at 1732277610260Running coprocessor pre-close hooks at 1732277610260Disabling compacts and flushes for region at 1732277610260Disabling writes for close at 1732277610260Writing region close event to WAL at 1732277610261 (+1 ms)Running coprocessor post-close hooks at 1732277610266 (+5 ms)Closed at 1732277610266 2024-11-22T12:13:30,267 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732277604754.142956bd7cdd41ce32216563195a828d. 2024-11-22T12:13:30,267 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4f3a8ed48d3b002e0140f5d1f4422c45, disabling compactions & flushes 2024-11-22T12:13:30,267 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:30,267 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:30,267 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. after waiting 0 ms 2024-11-22T12:13:30,267 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:30,269 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/.tmp/info/570f02170023490181fcfa97e06a0c30 as hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/info/570f02170023490181fcfa97e06a0c30 2024-11-22T12:13:30,272 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4f3a8ed48d3b002e0140f5d1f4422c45/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,272 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/eb5b2e2a4461c2824dc11a5a423a6d19/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,272 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:30,272 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4f3a8ed48d3b002e0140f5d1f4422c45: Waiting for close lock at 1732277610267Running coprocessor pre-close hooks at 1732277610267Disabling compacts and flushes for region at 1732277610267Disabling writes for close at 1732277610267Writing region close event to WAL at 1732277610267Running coprocessor post-close hooks at 1732277610272 (+5 ms)Closed at 1732277610272 2024-11-22T12:13:30,272 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb5b2e2a4461c2824dc11a5a423a6d19: Waiting for close lock at 1732277610266Running coprocessor pre-close hooks at 1732277610266Disabling compacts and flushes for region at 1732277610266Disabling writes for close at 1732277610266Writing region close event to WAL at 1732277610267 (+1 ms)Running coprocessor post-close hooks at 1732277610272 (+5 ms)Closed at 1732277610272 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732277604754.4f3a8ed48d3b002e0140f5d1f4422c45. 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732277604754.eb5b2e2a4461c2824dc11a5a423a6d19. 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7aef6551d501af493aaa4fd1077bf9c9, disabling compactions & flushes 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d1e0cdf60c0d514100f271449411b4ed, disabling compactions & flushes 2024-11-22T12:13:30,273 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:30,273 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. after waiting 0 ms 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. after waiting 0 ms 2024-11-22T12:13:30,273 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:30,278 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7aef6551d501af493aaa4fd1077bf9c9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,278 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:30,279 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7aef6551d501af493aaa4fd1077bf9c9: Waiting for close lock at 1732277610273Running coprocessor pre-close hooks at 1732277610273Disabling compacts and flushes for region at 1732277610273Disabling writes for close at 1732277610273Writing region close event to WAL at 1732277610273Running coprocessor post-close hooks at 1732277610278 (+5 ms)Closed at 1732277610278 2024-11-22T12:13:30,279 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732277604754.7aef6551d501af493aaa4fd1077bf9c9. 2024-11-22T12:13:30,279 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1e0cdf60c0d514100f271449411b4ed/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,279 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88ee6f91206e330acd4027ac1712c4b5, disabling compactions & flushes 2024-11-22T12:13:30,279 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:30,279 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:30,279 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. after waiting 0 ms 2024-11-22T12:13:30,279 INFO [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/info/570f02170023490181fcfa97e06a0c30, entries=24, sequenceid=71, filesize=8.0 K 2024-11-22T12:13:30,279 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:30,280 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:30,280 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d1e0cdf60c0d514100f271449411b4ed: Waiting for close lock at 1732277610273Running coprocessor pre-close hooks at 1732277610273Disabling compacts and flushes for region at 1732277610273Disabling writes for close at 1732277610273Writing region close event to WAL at 1732277610273Running coprocessor post-close hooks at 1732277610280 (+7 ms)Closed at 1732277610280 2024-11-22T12:13:30,280 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732277604754.d1e0cdf60c0d514100f271449411b4ed. 2024-11-22T12:13:30,280 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4c61ef28412030a5de1748e0494b5d77, disabling compactions & flushes 2024-11-22T12:13:30,280 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:30,280 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:30,280 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. after waiting 0 ms 2024-11-22T12:13:30,280 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:30,281 INFO [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 65ms, sequenceid=71, compaction requested=false 2024-11-22T12:13:30,283 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88ee6f91206e330acd4027ac1712c4b5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,284 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:30,284 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88ee6f91206e330acd4027ac1712c4b5: Waiting for close lock at 1732277610279Running coprocessor pre-close hooks at 1732277610279Disabling compacts and flushes for region at 1732277610279Disabling writes for close at 1732277610279Writing region close event to WAL at 1732277610280 (+1 ms)Running coprocessor post-close hooks at 1732277610284 (+4 ms)Closed at 1732277610284 2024-11-22T12:13:30,284 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732277604754.88ee6f91206e330acd4027ac1712c4b5. 2024-11-22T12:13:30,285 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/4c61ef28412030a5de1748e0494b5d77/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,285 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b4584ab1e18d121599a608cdb4be19ea, disabling compactions & flushes 2024-11-22T12:13:30,285 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:30,285 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:30,285 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. after waiting 0 ms 2024-11-22T12:13:30,285 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:30,285 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:30,285 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4c61ef28412030a5de1748e0494b5d77: Waiting for close lock at 1732277610280Running coprocessor pre-close hooks at 1732277610280Disabling compacts and flushes for region at 1732277610280Disabling writes for close at 1732277610280Writing region close event to WAL at 1732277610281 (+1 ms)Running coprocessor post-close hooks at 1732277610285 (+4 ms)Closed at 1732277610285 2024-11-22T12:13:30,285 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732277604754.4c61ef28412030a5de1748e0494b5d77. 2024-11-22T12:13:30,285 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e33fb94d3dd5df2c85edbc81c792b792, disabling compactions & flushes 2024-11-22T12:13:30,285 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:30,285 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:30,286 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. after waiting 0 ms 2024-11-22T12:13:30,286 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:30,289 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-22T12:13:30,289 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b4584ab1e18d121599a608cdb4be19ea/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,290 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T12:13:30,290 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:30,290 INFO [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T12:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b4584ab1e18d121599a608cdb4be19ea: Waiting for close lock at 1732277610285Running coprocessor pre-close hooks at 1732277610285Disabling compacts and flushes for region at 1732277610285Disabling writes for close at 1732277610285Writing region close event to WAL at 1732277610285Running coprocessor post-close hooks at 1732277610290 (+5 ms)Closed at 1732277610290 2024-11-22T12:13:30,290 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732277610216Running coprocessor pre-close hooks at 1732277610216Disabling compacts and flushes for region at 1732277610216Disabling writes for close at 1732277610216Obtaining lock to block concurrent updates at 1732277610216Preparing flush snapshotting stores in 1588230740 at 1732277610216Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732277610223 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732277610225 (+2 ms)Flushing 1588230740/info: creating writer at 1732277610226 (+1 ms)Flushing 1588230740/info: appending metadata at 1732277610234 (+8 ms)Flushing 1588230740/info: closing flushed file at 1732277610234Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c2bd52f: reopening flushed file at 1732277610268 (+34 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 65ms, sequenceid=71, compaction requested=false at 1732277610281 (+13 ms)Writing region close event to WAL at 1732277610285 (+4 ms)Running coprocessor post-close hooks at 1732277610290 (+5 ms)Closed at 1732277610290 2024-11-22T12:13:30,290 DEBUG [RS_CLOSE_META-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T12:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732277604754.b4584ab1e18d121599a608cdb4be19ea. 2024-11-22T12:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7cf86cd087fe778d3cb9ffcdb4f32fdf, disabling compactions & flushes 2024-11-22T12:13:30,290 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. after waiting 0 ms 2024-11-22T12:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:30,294 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/7cf86cd087fe778d3cb9ffcdb4f32fdf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,295 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:30,295 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7cf86cd087fe778d3cb9ffcdb4f32fdf: Waiting for close lock at 1732277610290Running coprocessor pre-close hooks at 1732277610290Disabling compacts and flushes for region at 1732277610290Disabling writes for close at 1732277610290Writing region close event to WAL at 1732277610291 (+1 ms)Running coprocessor post-close hooks at 1732277610295 (+4 ms)Closed at 1732277610295 2024-11-22T12:13:30,295 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732277604754.7cf86cd087fe778d3cb9ffcdb4f32fdf. 2024-11-22T12:13:30,295 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cca3907a8067af4c43fffaebaa942770, disabling compactions & flushes 2024-11-22T12:13:30,295 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:30,295 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:30,295 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. after waiting 0 ms 2024-11-22T12:13:30,295 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:30,297 INFO [regionserver/46e18b47f3d3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T12:13:30,300 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/e33fb94d3dd5df2c85edbc81c792b792/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,300 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/cca3907a8067af4c43fffaebaa942770/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,301 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:30,301 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cca3907a8067af4c43fffaebaa942770: Waiting for close lock at 1732277610295Running coprocessor pre-close hooks at 1732277610295Disabling compacts and flushes for region at 1732277610295Disabling writes for close at 1732277610295Writing region close event to WAL at 1732277610296 (+1 ms)Running coprocessor post-close hooks at 1732277610301 (+5 ms)Closed at 1732277610301 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e33fb94d3dd5df2c85edbc81c792b792: Waiting for close lock at 1732277610285Running coprocessor pre-close hooks at 1732277610285Disabling compacts and flushes for region at 1732277610285Disabling writes for close at 1732277610286 (+1 ms)Writing region close event to WAL at 1732277610286Running coprocessor post-close hooks at 1732277610301 (+15 ms)Closed at 1732277610301 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732277604754.cca3907a8067af4c43fffaebaa942770. 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732277604754.e33fb94d3dd5df2c85edbc81c792b792. 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b745bd27cbcb9cc89fc04815d334ae24, disabling compactions & flushes 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d1716ab8b0523ebd9a28d11dac1fd8e6, disabling compactions & flushes 2024-11-22T12:13:30,301 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:30,301 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. after waiting 0 ms 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. after waiting 0 ms 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:30,301 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:30,305 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/d1716ab8b0523ebd9a28d11dac1fd8e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,305 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/b745bd27cbcb9cc89fc04815d334ae24/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,306 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:30,306 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:30,306 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d1716ab8b0523ebd9a28d11dac1fd8e6: Waiting for close lock at 1732277610301Running coprocessor pre-close hooks at 1732277610301Disabling compacts and flushes for region at 1732277610301Disabling writes for close at 1732277610301Writing region close event to WAL at 1732277610301Running coprocessor post-close hooks at 1732277610306 (+5 ms)Closed at 1732277610306 2024-11-22T12:13:30,306 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b745bd27cbcb9cc89fc04815d334ae24: Waiting for close lock at 1732277610301Running coprocessor pre-close hooks at 1732277610301Disabling compacts and flushes for region at 1732277610301Disabling writes for close at 1732277610301Writing region close event to WAL at 1732277610301Running coprocessor post-close hooks at 1732277610306 (+5 ms)Closed at 1732277610306 2024-11-22T12:13:30,306 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732277604754.d1716ab8b0523ebd9a28d11dac1fd8e6. 2024-11-22T12:13:30,306 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732277604754.b745bd27cbcb9cc89fc04815d334ae24. 2024-11-22T12:13:30,306 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88a8bf4fbe7648785154bd80df699f0b, disabling compactions & flushes 2024-11-22T12:13:30,306 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:30,306 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:30,306 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. after waiting 0 ms 2024-11-22T12:13:30,306 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:30,310 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/data/default/test/88a8bf4fbe7648785154bd80df699f0b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T12:13:30,311 INFO [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:30,311 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88a8bf4fbe7648785154bd80df699f0b: Waiting for close lock at 1732277610306Running coprocessor pre-close hooks at 1732277610306Disabling compacts and flushes for region at 1732277610306Disabling writes for close at 1732277610306Writing region close event to WAL at 1732277610307 (+1 ms)Running coprocessor post-close hooks at 1732277610311 (+4 ms)Closed at 1732277610311 2024-11-22T12:13:30,311 DEBUG [RS_CLOSE_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732277604754.88a8bf4fbe7648785154bd80df699f0b. 2024-11-22T12:13:30,416 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(976): stopping server 46e18b47f3d3,40513,1732277601956; all regions closed. 2024-11-22T12:13:30,416 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(976): stopping server 46e18b47f3d3,38629,1732277601859; all regions closed. 2024-11-22T12:13:30,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741835_1011 (size=6805) 2024-11-22T12:13:30,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741835_1011 (size=6805) 2024-11-22T12:13:30,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741865_1041 (size=4204) 2024-11-22T12:13:30,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741865_1041 (size=4204) 2024-11-22T12:13:30,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741865_1041 (size=4204) 2024-11-22T12:13:30,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741835_1011 (size=6805) 2024-11-22T12:13:30,423 DEBUG [RS:0;46e18b47f3d3:38629 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs 2024-11-22T12:13:30,423 DEBUG [RS:1;46e18b47f3d3:40513 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs 2024-11-22T12:13:30,423 INFO [RS:0;46e18b47f3d3:38629 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 46e18b47f3d3%2C38629%2C1732277601859.meta:.meta(num 1732277608061) 2024-11-22T12:13:30,423 INFO [RS:1;46e18b47f3d3:40513 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 46e18b47f3d3%2C40513%2C1732277601956:(num 1732277603735) 2024-11-22T12:13:30,423 DEBUG [RS:1;46e18b47f3d3:40513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:30,424 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T12:13:30,424 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T12:13:30,424 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.ChoreService(370): Chore service for: regionserver/46e18b47f3d3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T12:13:30,424 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T12:13:30,424 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T12:13:30,424 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T12:13:30,424 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T12:13:30,425 INFO [RS:1;46e18b47f3d3:40513 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40513 2024-11-22T12:13:30,425 INFO [regionserver/46e18b47f3d3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T12:13:30,425 WARN [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T12:13:30,425 WARN [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5., aborting... java.io.IOException: Failed to report opened region to master: test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T12:13:30,426 ERROR [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2440): ***** ABORTING region server 46e18b47f3d3,40513,1732277601956: Failed to open region test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,lll,1732277604754.c3b4530e36a471e0bfd4c64c45ec26d5. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T12:13:30,426 ERROR [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-22T12:13:30,426 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-22T12:13:30,427 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-22T12:13:30,430 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-22T12:13:30,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741834_1010 (size=6795) 2024-11-22T12:13:30,432 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-22T12:13:30,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741834_1010 (size=6795) 2024-11-22T12:13:30,441 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 478076496 }, "NonHeapMemoryUsage": { "committed": 163315712, "init": 7667712, "max": -1, "used": 160805264 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "46e18b47f3d3", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40153, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 128, "ProcessCallTime_min": 0, "ProcessCallTime_max": 23, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 5, "ProcessCallTime_median": 11, "ProcessCallTime_75th_percentile": 17, "ProcessCallTime_90th_percentile": 20, "ProcessCallTime_95th_percentile": 21, "ProcessCallTime_98th_percentile": 22, "ProcessCallTime_99th_percentile": 22, "ProcessCallTime_99.9th_percentile": 22, "ProcessCallTime_TimeRangeCount_0-1": 128, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 128, "QueueCallTime_min": 0, "QueueCallTime_max": 12, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 6, "QueueCallTime_75th_percentile": 9, "QueueCallTime_90th_percentile": 10, "QueueCallTime_95th_percentile": 11, "QueueCallTime_98th_percentile": 11, "QueueCallTime_99th_percentile": 11, "QueueCallTime_99.9th_percentile": 11, "QueueCallTime_TimeRangeCount_0-1": 128, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 128, "TotalCallTime_min": 0, "TotalCallTime_max": 24, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 6, "TotalCallTime_median": 12, "TotalCallTime_75th_percentile": 18, "TotalCallTime_90th_percentile": 21, "TotalCallTime_95th_percentile": 22, "TotalCallTime_98th_percentile": 23, "TotalCallTime_99th_percentile": 23, "TotalCallTime_99.9th_percentile": 23, "TotalCallTime_TimeRangeCount_0-1": 128, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 128, "ResponseSize_min": 0, "ResponseSize_max": 22854, "ResponseSize_mean": 3926, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17140, "ResponseSize_90th_percentile": 20568, "ResponseSize_95th_percentile": 21711, "ResponseSize_98th_percentile": 22396, "ResponseSize_99th_percentile": 22625, "ResponseSize_99.9th_percentile": 22831, "ResponseSize_SizeRangeCount_0-10": 128, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 128, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 284, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 128, "sentBytes": 389456 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "46e18b47f3d3", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59102", "tag.serverName": "46e18b47f3d3,38629,1732277601859", "tag.clusterId": "5caf31ec-e0fb-4464-8e14-cb86b3459b61", "tag.Context": "regionserver", "tag.Hostname": "46e18b47f3d3", "regionCount": 13, "storeCount": 13, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 3, "maxStoreFileCount": 1, "memStoreSize": 0, "memStoreHeapSize": 3328, "memStoreOffHeapSize": 0, "storeFileSize": 45242, "storeFileSizeGrowthRate": 9048.4, "maxStoreFileAge": 290, "minStoreFileAge": 188, "avgStoreFileAge": 234, "numReferenceFiles": 0, "regionServerStartTime": 1732277601859, "averageRegionSize": 4524, "storeFileIndexSize": 1648, "staticIndexSize": 442, "staticBloomSize": 264, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922038576, "blockCacheCount": 4, "blockCacheDataBlockCount": 4, "blockCacheSize": 708304, "blockCacheCountHitPercent": 94.73684210526315, "blockCacheExpressHitPercent": 94.73684210526315, "l1CacheSize": 708304, "l1CacheFreeSize": 922038576, "l1CacheCount": 4, "l1CacheEvictionCount": 0, "l1CacheHitCount": 72, "l1CacheMissCount": 4, "l1CacheHitRatio": 0.9473684210526315, "l1CacheMissRatio": 0.05263157894736842, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 1264640, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 36, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 28, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 72, "blockCacheHitCountPrimary": 72, "blockCacheHitCachingCount": 72, "blockCacheMissCount": 4, "blockCacheMissCountPrimary": 4, "blockCacheMissCachingCount": 4, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 2, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 36, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 264, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 28672, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 28, "ScanTime_min": 0, "ScanTime_max": 13, "ScanTime_mean": 2, "ScanTime_25th_percentile": 3, "ScanTime_median": 6, "ScanTime_75th_percentile": 9, "ScanTime_90th_percentile": 11, "ScanTime_95th_percentile": 12, "ScanTime_98th_percentile": 12, "ScanTime_99th_percentile": 12, "ScanTime_99.9th_percentile": 12, "ScanTime_TimeRangeCount_0-1": 28, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 1, "Put_max": 1, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 1, "Put_75th_percentile": 1, "Put_90th_percentile": 1, "Put_95th_percentile": 1, "Put_98th_percentile": 1, "Put_99th_percentile": 1, "Put_99.9th_percentile": 1, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 45242, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 28672, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 45242, "FlushOutputSize_max": 45242, "FlushOutputSize_mean": 45242, "FlushOutputSize_25th_percentile": 45242, "FlushOutputSize_median": 45242, "FlushOutputSize_75th_percentile": 45242, "FlushOutputSize_90th_percentile": 45242, "FlushOutputSize_95th_percentile": 45242, "FlushOutputSize_98th_percentile": 45242, "FlushOutputSize_99th_percentile": 45242, "FlushOutputSize_99.9th_percentile": 45242, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 19, "ScanBlockBytesScanned_min": 28942, "ScanBlockBytesScanned_max": 28942, "ScanBlockBytesScanned_mean": 28942, "ScanBlockBytesScanned_25th_percentile": 28942, "ScanBlockBytesScanned_median": 28942, "ScanBlockBytesScanned_75th_percentile": 28942, "ScanBlockBytesScanned_90th_percentile": 28942, "ScanBlockBytesScanned_95th_percentile": 28942, "ScanBlockBytesScanned_98th_percentile": 28942, "ScanBlockBytesScanned_99th_percentile": 28942, "ScanBlockBytesScanned_99.9th_percentile": 28942, "ScanBlockBytesScanned_SizeRangeCount_0-10": 19, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 549898, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 28672, "FlushMemstoreSize_max": 28672, "FlushMemstoreSize_mean": 28672, "FlushMemstoreSize_25th_percentile": 28672, "FlushMemstoreSize_median": 28672, "FlushMemstoreSize_75th_percentile": 28672, "FlushMemstoreSize_90th_percentile": 28672, "FlushMemstoreSize_95th_percentile": 28672, "FlushMemstoreSize_98th_percentile": 28672, "FlushMemstoreSize_99th_percentile": 28672, "FlushMemstoreSize_99.9th_percentile": 28672, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 237, "FlushTime_max": 237, "FlushTime_mean": 237, "FlushTime_25th_percentile": 237, "FlushTime_median": 237, "FlushTime_75th_percentile": 237, "FlushTime_90th_percentile": 237, "FlushTime_95th_percentile": 237, "FlushTime_98th_percentile": 237, "FlushTime_99th_percentile": 237, "FlushTime_99.9th_percentile": 237, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 28, "ScanSize_min": 0, "ScanSize_max": 22728, "ScanSize_mean": 15422, "ScanSize_25th_percentile": 5682, "ScanSize_median": 11364, "ScanSize_75th_percentile": 17046, "ScanSize_90th_percentile": 20455, "ScanSize_95th_percentile": 21591, "ScanSize_98th_percentile": 22273, "ScanSize_99th_percentile": 22500, "ScanSize_99.9th_percentile": 22705, "ScanSize_SizeRangeCount_0-10": 28, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-22T12:13:30,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/46e18b47f3d3,40513,1732277601956 2024-11-22T12:13:30,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741834_1010 (size=6795) 2024-11-22T12:13:30,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T12:13:30,445 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T12:13:30,448 DEBUG [RS:0;46e18b47f3d3:38629 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/oldWALs 2024-11-22T12:13:30,449 INFO [RS:0;46e18b47f3d3:38629 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 46e18b47f3d3%2C38629%2C1732277601859:(num 1732277603706) 2024-11-22T12:13:30,449 DEBUG [RS:0;46e18b47f3d3:38629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T12:13:30,449 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T12:13:30,449 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T12:13:30,449 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.ChoreService(370): Chore service for: regionserver/46e18b47f3d3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T12:13:30,449 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T12:13:30,449 INFO [regionserver/46e18b47f3d3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T12:13:30,449 INFO [RS:0;46e18b47f3d3:38629 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38629 2024-11-22T12:13:30,449 WARN [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T12:13:30,450 WARN [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4., aborting... java.io.IOException: Failed to report opened region to master: test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T12:13:30,450 ERROR [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2440): ***** ABORTING region server 46e18b47f3d3,38629,1732277601859: Failed to open region test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,iii,1732277604754.d5254a456e6265d346df2dc9fbb1afe4. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T12:13:30,450 ERROR [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-22T12:13:30,450 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-22T12:13:30,451 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-22T12:13:30,452 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-22T12:13:30,452 DEBUG [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-22T12:13:30,454 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [46e18b47f3d3,40513,1732277601956] 2024-11-22T12:13:30,456 INFO [RS_OPEN_REGION-regionserver/46e18b47f3d3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 480173648 }, "NonHeapMemoryUsage": { "committed": 163577856, "init": 7667712, "max": -1, "used": 161051904 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "46e18b47f3d3", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40153, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 128, "ProcessCallTime_min": 0, "ProcessCallTime_max": 23, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 5, "ProcessCallTime_median": 11, "ProcessCallTime_75th_percentile": 17, "ProcessCallTime_90th_percentile": 20, "ProcessCallTime_95th_percentile": 21, "ProcessCallTime_98th_percentile": 22, "ProcessCallTime_99th_percentile": 22, "ProcessCallTime_99.9th_percentile": 22, "ProcessCallTime_TimeRangeCount_0-1": 128, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 128, "QueueCallTime_min": 0, "QueueCallTime_max": 12, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 6, "QueueCallTime_75th_percentile": 9, "QueueCallTime_90th_percentile": 10, "QueueCallTime_95th_percentile": 11, "QueueCallTime_98th_percentile": 11, "QueueCallTime_99th_percentile": 11, "QueueCallTime_99.9th_percentile": 11, "QueueCallTime_TimeRangeCount_0-1": 128, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 128, "TotalCallTime_min": 0, "TotalCallTime_max": 24, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 6, "TotalCallTime_median": 12, "TotalCallTime_75th_percentile": 18, "TotalCallTime_90th_percentile": 21, "TotalCallTime_95th_percentile": 22, "TotalCallTime_98th_percentile": 23, "TotalCallTime_99th_percentile": 23, "TotalCallTime_99.9th_percentile": 23, "TotalCallTime_TimeRangeCount_0-1": 128, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 128, "ResponseSize_min": 0, "ResponseSize_max": 22854, "ResponseSize_mean": 3926, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17140, "ResponseSize_90th_percentile": 20568, "ResponseSize_95th_percentile": 21711, "ResponseSize_98th_percentile": 22396, "ResponseSize_99th_percentile": 22625, "ResponseSize_99.9th_percentile": 22831, "ResponseSize_SizeRangeCount_0-10": 128, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 128, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 284, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 128, "sentBytes": 389456 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "46e18b47f3d3", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59102", "tag.serverName": "46e18b47f3d3,38629,1732277601859", "tag.clusterId": "5caf31ec-e0fb-4464-8e14-cb86b3459b61", "tag.Context": "regionserver", "tag.Hostname": "46e18b47f3d3", "regionCount": 13, "storeCount": 13, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 3, "maxStoreFileCount": 1, "memStoreSize": 0, "memStoreHeapSize": 3328, "memStoreOffHeapSize": 0, "storeFileSize": 45242, "storeFileSizeGrowthRate": 9048.4, "maxStoreFileAge": 290, "minStoreFileAge": 188, "avgStoreFileAge": 234, "numReferenceFiles": 0, "regionServerStartTime": 1732277601859, "averageRegionSize": 4524, "storeFileIndexSize": 1648, "staticIndexSize": 442, "staticBloomSize": 264, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922038576, "blockCacheCount": 4, "blockCacheDataBlockCount": 4, "blockCacheSize": 708304, "blockCacheCountHitPercent": 94.73684210526315, "blockCacheExpressHitPercent": 94.73684210526315, "l1CacheSize": 708304, "l1CacheFreeSize": 922038576, "l1CacheCount": 4, "l1CacheEvictionCount": 0, "l1CacheHitCount": 72, "l1CacheMissCount": 4, "l1CacheHitRatio": 0.9473684210526315, "l1CacheMissRatio": 0.05263157894736842, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 1264640, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 36, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 28, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 72, "blockCacheHitCountPrimary": 72, "blockCacheHitCachingCount": 72, "blockCacheMissCount": 4, "blockCacheMissCountPrimary": 4, "blockCacheMissCachingCount": 4, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 2, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 36, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 264, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 28672, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 28, "ScanTime_min": 0, "ScanTime_max": 13, "ScanTime_mean": 2, "ScanTime_25th_percentile": 3, "ScanTime_median": 6, "ScanTime_75th_percentile": 9, "ScanTime_90th_percentile": 11, "ScanTime_95th_percentile": 12, "ScanTime_98th_percentile": 12, "ScanTime_99th_percentile": 12, "ScanTime_99.9th_percentile": 12, "ScanTime_TimeRangeCount_0-1": 28, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 1, "Put_max": 1, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 1, "Put_75th_percentile": 1, "Put_90th_percentile": 1, "Put_95th_percentile": 1, "Put_98th_percentile": 1, "Put_99th_percentile": 1, "Put_99.9th_percentile": 1, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 45242, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 28672, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 45242, "FlushOutputSize_max": 45242, "FlushOutputSize_mean": 45242, "FlushOutputSize_25th_percentile": 45242, "FlushOutputSize_median": 45242, "FlushOutputSize_75th_percentile": 45242, "FlushOutputSize_90th_percentile": 45242, "FlushOutputSize_95th_percentile": 45242, "FlushOutputSize_98th_percentile": 45242, "FlushOutputSize_99th_percentile": 45242, "FlushOutputSize_99.9th_percentile": 45242, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 19, "ScanBlockBytesScanned_min": 28942, "ScanBlockBytesScanned_max": 28942, "ScanBlockBytesScanned_mean": 28942, "ScanBlockBytesScanned_25th_percentile": 28942, "ScanBlockBytesScanned_median": 28942, "ScanBlockBytesScanned_75th_percentile": 28942, "ScanBlockBytesScanned_90th_percentile": 28942, "ScanBlockBytesScanned_95th_percentile": 28942, "ScanBlockBytesScanned_98th_percentile": 28942, "ScanBlockBytesScanned_99th_percentile": 28942, "ScanBlockBytesScanned_99.9th_percentile": 28942, "ScanBlockBytesScanned_SizeRangeCount_0-10": 19, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 549898, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 28672, "FlushMemstoreSize_max": 28672, "FlushMemstoreSize_mean": 28672, "FlushMemstoreSize_25th_percentile": 28672, "FlushMemstoreSize_median": 28672, "FlushMemstoreSize_75th_percentile": 28672, "FlushMemstoreSize_90th_percentile": 28672, "FlushMemstoreSize_95th_percentile": 28672, "FlushMemstoreSize_98th_percentile": 28672, "FlushMemstoreSize_99th_percentile": 28672, "FlushMemstoreSize_99.9th_percentile": 28672, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 237, "FlushTime_max": 237, "FlushTime_mean": 237, "FlushTime_25th_percentile": 237, "FlushTime_median": 237, "FlushTime_75th_percentile": 237, "FlushTime_90th_percentile": 237, "FlushTime_95th_percentile": 237, "FlushTime_98th_percentile": 237, "FlushTime_99th_percentile": 237, "FlushTime_99.9th_percentile": 237, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 28, "ScanSize_min": 0, "ScanSize_max": 22728, "ScanSize_mean": 15422, "ScanSize_25th_percentile": 5682, "ScanSize_median": 11364, "ScanSize_75th_percentile": 17046, "ScanSize_90th_percentile": 20455, "ScanSize_95th_percentile": 21591, "ScanSize_98th_percentile": 22273, "ScanSize_99th_percentile": 22500, "ScanSize_99.9th_percentile": 22705, "ScanSize_SizeRangeCount_0-10": 28, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-22T12:13:30,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/46e18b47f3d3,38629,1732277601859 2024-11-22T12:13:30,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T12:13:30,464 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T12:13:30,475 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/46e18b47f3d3,40513,1732277601956 already deleted, retry=false 2024-11-22T12:13:30,475 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 46e18b47f3d3,40513,1732277601956 expired; onlineServers=1 2024-11-22T12:13:30,485 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [46e18b47f3d3,38629,1732277601859] 2024-11-22T12:13:30,495 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/46e18b47f3d3,38629,1732277601859 already deleted, retry=false 2024-11-22T12:13:30,496 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 46e18b47f3d3,38629,1732277601859 expired; onlineServers=0 2024-11-22T12:13:30,496 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '46e18b47f3d3,35109,1732277601092' ***** 2024-11-22T12:13:30,496 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T12:13:30,496 INFO [M:0;46e18b47f3d3:35109 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T12:13:30,496 INFO [M:0;46e18b47f3d3:35109 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T12:13:30,496 DEBUG [M:0;46e18b47f3d3:35109 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T12:13:30,496 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T12:13:30,496 DEBUG [M:0;46e18b47f3d3:35109 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T12:13:30,496 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster-HFileCleaner.large.0-1732277603363 {}] cleaner.HFileCleaner(306): Exit Thread[master/46e18b47f3d3:0:becomeActiveMaster-HFileCleaner.large.0-1732277603363,5,FailOnTimeoutGroup] 2024-11-22T12:13:30,496 DEBUG [master/46e18b47f3d3:0:becomeActiveMaster-HFileCleaner.small.0-1732277603365 {}] cleaner.HFileCleaner(306): Exit Thread[master/46e18b47f3d3:0:becomeActiveMaster-HFileCleaner.small.0-1732277603365,5,FailOnTimeoutGroup] 2024-11-22T12:13:30,497 INFO [M:0;46e18b47f3d3:35109 {}] hbase.ChoreService(370): Chore service for: master/46e18b47f3d3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T12:13:30,497 INFO [M:0;46e18b47f3d3:35109 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T12:13:30,497 DEBUG [M:0;46e18b47f3d3:35109 {}] master.HMaster(1795): Stopping service threads 2024-11-22T12:13:30,497 INFO [M:0;46e18b47f3d3:35109 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T12:13:30,497 INFO [M:0;46e18b47f3d3:35109 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T12:13:30,497 INFO [M:0;46e18b47f3d3:35109 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T12:13:30,497 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T12:13:30,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T12:13:30,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T12:13:30,506 DEBUG [M:0;46e18b47f3d3:35109 {}] zookeeper.ZKUtil(347): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T12:13:30,506 WARN [M:0;46e18b47f3d3:35109 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T12:13:30,507 INFO [M:0;46e18b47f3d3:35109 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/.lastflushedseqids 2024-11-22T12:13:30,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741870_1046 (size=1) 2024-11-22T12:13:30,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741870_1046 (size=1) 2024-11-22T12:13:30,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741870_1046 (size=1) 2024-11-22T12:13:30,520 INFO [M:0;46e18b47f3d3:35109 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T12:13:30,520 INFO [M:0;46e18b47f3d3:35109 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T12:13:30,520 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T12:13:30,520 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T12:13:30,520 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T12:13:30,520 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T12:13:30,520 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T12:13:30,521 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=249.43 KB heapSize=303.12 KB 2024-11-22T12:13:30,539 DEBUG [M:0;46e18b47f3d3:35109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ef0e508b679841d19d5d738fe9a67364 is 82, key is hbase:meta,,1/info:regioninfo/1732277608383/Put/seqid=0 2024-11-22T12:13:30,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741871_1047 (size=6071) 2024-11-22T12:13:30,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741871_1047 (size=6071) 2024-11-22T12:13:30,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741871_1047 (size=6071) 2024-11-22T12:13:30,551 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ef0e508b679841d19d5d738fe9a67364 2024-11-22T12:13:30,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T12:13:30,554 INFO [RS:1;46e18b47f3d3:40513 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T12:13:30,554 INFO [RS:1;46e18b47f3d3:40513 {}] regionserver.HRegionServer(1031): Exiting; stopping=46e18b47f3d3,40513,1732277601956; zookeeper connection closed. 2024-11-22T12:13:30,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40513-0x10162767bb80002, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T12:13:30,554 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@725a6a70 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@725a6a70 2024-11-22T12:13:30,577 DEBUG [M:0;46e18b47f3d3:35109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ac9242d1e0d40d882c93288b6f12a0a is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732277610183/Put/seqid=0 2024-11-22T12:13:30,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741872_1048 (size=12800) 2024-11-22T12:13:30,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741872_1048 (size=12800) 2024-11-22T12:13:30,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741872_1048 (size=12800) 2024-11-22T12:13:30,585 INFO [RS:0;46e18b47f3d3:38629 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T12:13:30,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T12:13:30,585 INFO [RS:0;46e18b47f3d3:38629 {}] regionserver.HRegionServer(1031): Exiting; stopping=46e18b47f3d3,38629,1732277601859; zookeeper connection closed. 2024-11-22T12:13:30,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38629-0x10162767bb80001, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T12:13:30,586 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@18a596db {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@18a596db 2024-11-22T12:13:30,586 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.16 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ac9242d1e0d40d882c93288b6f12a0a 2024-11-22T12:13:30,586 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-22T12:13:30,607 DEBUG [M:0;46e18b47f3d3:35109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c8f7f790f8954078a61598292445d4df is 69, key is 46e18b47f3d3,38629,1732277601859/rs:state/1732277603374/Put/seqid=0 2024-11-22T12:13:30,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741873_1049 (size=5442) 2024-11-22T12:13:30,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741873_1049 (size=5442) 2024-11-22T12:13:30,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741873_1049 (size=5442) 2024-11-22T12:13:30,617 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c8f7f790f8954078a61598292445d4df 2024-11-22T12:13:30,623 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8f7f790f8954078a61598292445d4df 2024-11-22T12:13:30,638 DEBUG [M:0;46e18b47f3d3:35109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cac2b5ec3c0d4136bde2995772a7c5fa is 52, key is load_balancer_on/state:d/1732277607104/Put/seqid=0 2024-11-22T12:13:30,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741874_1050 (size=5058) 2024-11-22T12:13:30,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741874_1050 (size=5058) 2024-11-22T12:13:30,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741874_1050 (size=5058) 2024-11-22T12:13:30,645 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cac2b5ec3c0d4136bde2995772a7c5fa 2024-11-22T12:13:30,652 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ef0e508b679841d19d5d738fe9a67364 as hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ef0e508b679841d19d5d738fe9a67364 2024-11-22T12:13:30,659 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ef0e508b679841d19d5d738fe9a67364, entries=14, sequenceid=573, filesize=5.9 K 2024-11-22T12:13:30,660 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ac9242d1e0d40d882c93288b6f12a0a as hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ac9242d1e0d40d882c93288b6f12a0a 2024-11-22T12:13:30,666 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ac9242d1e0d40d882c93288b6f12a0a, entries=79, sequenceid=573, filesize=12.5 K 2024-11-22T12:13:30,667 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c8f7f790f8954078a61598292445d4df as hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c8f7f790f8954078a61598292445d4df 2024-11-22T12:13:30,675 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8f7f790f8954078a61598292445d4df 2024-11-22T12:13:30,675 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c8f7f790f8954078a61598292445d4df, entries=3, sequenceid=573, filesize=5.3 K 2024-11-22T12:13:30,676 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cac2b5ec3c0d4136bde2995772a7c5fa as hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cac2b5ec3c0d4136bde2995772a7c5fa 2024-11-22T12:13:30,685 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36765/user/jenkins/test-data/1ab99b02-c270-62a6-cd1c-e87eadcb149c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cac2b5ec3c0d4136bde2995772a7c5fa, entries=1, sequenceid=573, filesize=4.9 K 2024-11-22T12:13:30,686 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.HRegion(3140): Finished flush of dataSize ~249.43 KB/255416, heapSize ~303.05 KB/310328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 166ms, sequenceid=573, compaction requested=false 2024-11-22T12:13:30,687 INFO [M:0;46e18b47f3d3:35109 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T12:13:30,687 DEBUG [M:0;46e18b47f3d3:35109 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732277610520Disabling compacts and flushes for region at 1732277610520Disabling writes for close at 1732277610520Obtaining lock to block concurrent updates at 1732277610521 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732277610521Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=255416, getHeapSize=310328, getOffHeapSize=0, getCellsCount=712 at 1732277610521Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732277610522 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732277610522Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732277610538 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732277610539 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732277610557 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732277610576 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732277610576Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732277610592 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732277610606 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732277610606Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732277610623 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732277610637 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732277610637Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3df846f6: reopening flushed file at 1732277610651 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4aa8e341: reopening flushed file at 1732277610659 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c3498b1: reopening flushed file at 1732277610666 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b154fcc: reopening flushed file at 1732277610675 (+9 ms)Finished flush of dataSize ~249.43 KB/255416, heapSize ~303.05 KB/310328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 166ms, sequenceid=573, compaction requested=false at 1732277610686 (+11 ms)Writing region close event to WAL at 1732277610687 (+1 ms)Closed at 1732277610687 2024-11-22T12:13:30,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741830_1006 (size=293670) 2024-11-22T12:13:30,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33883 is added to blk_1073741830_1006 (size=293670) 2024-11-22T12:13:30,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741830_1006 (size=293670) 2024-11-22T12:13:30,691 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T12:13:30,691 INFO [M:0;46e18b47f3d3:35109 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T12:13:30,691 INFO [M:0;46e18b47f3d3:35109 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35109 2024-11-22T12:13:30,692 INFO [M:0;46e18b47f3d3:35109 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T12:13:30,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T12:13:30,796 INFO [M:0;46e18b47f3d3:35109 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T12:13:30,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35109-0x10162767bb80000, quorum=127.0.0.1:59102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T12:13:30,802 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6eecab6b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T12:13:30,805 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3fb3fd4b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T12:13:30,805 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T12:13:30,805 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75445ba5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T12:13:30,806 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@469f5a42{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.log.dir/,STOPPED} 2024-11-22T12:13:30,809 WARN [BP-1283605086-172.17.0.2-1732277596080 heartbeating to localhost/127.0.0.1:36765 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T12:13:30,809 WARN [BP-1283605086-172.17.0.2-1732277596080 heartbeating to localhost/127.0.0.1:36765 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1283605086-172.17.0.2-1732277596080 (Datanode Uuid 4d200714-a1ac-4dbf-b045-a6021c003158) service to localhost/127.0.0.1:36765 2024-11-22T12:13:30,809 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T12:13:30,809 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T12:13:30,810 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data5/current/BP-1283605086-172.17.0.2-1732277596080 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T12:13:30,810 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data6/current/BP-1283605086-172.17.0.2-1732277596080 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T12:13:30,810 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T12:13:30,812 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7549e2f1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T12:13:30,812 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@53e0679{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T12:13:30,812 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T12:13:30,812 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@540e5e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T12:13:30,812 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b3caf0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.log.dir/,STOPPED} 2024-11-22T12:13:30,813 WARN [BP-1283605086-172.17.0.2-1732277596080 heartbeating to localhost/127.0.0.1:36765 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T12:13:30,813 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T12:13:30,813 WARN [BP-1283605086-172.17.0.2-1732277596080 heartbeating to localhost/127.0.0.1:36765 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1283605086-172.17.0.2-1732277596080 (Datanode Uuid 61144862-4f8d-45c6-bb17-cf3221e76363) service to localhost/127.0.0.1:36765 2024-11-22T12:13:30,813 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T12:13:30,814 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data3/current/BP-1283605086-172.17.0.2-1732277596080 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T12:13:30,814 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data4/current/BP-1283605086-172.17.0.2-1732277596080 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T12:13:30,814 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T12:13:30,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4f33137f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T12:13:30,816 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6de9cba2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T12:13:30,816 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T12:13:30,817 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5026a795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T12:13:30,817 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20ec9bac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.log.dir/,STOPPED} 2024-11-22T12:13:30,818 WARN [BP-1283605086-172.17.0.2-1732277596080 heartbeating to localhost/127.0.0.1:36765 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T12:13:30,818 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T12:13:30,818 WARN [BP-1283605086-172.17.0.2-1732277596080 heartbeating to localhost/127.0.0.1:36765 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1283605086-172.17.0.2-1732277596080 (Datanode Uuid 513afaf4-2a3f-4084-895f-0011386d0f5c) service to localhost/127.0.0.1:36765 2024-11-22T12:13:30,818 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T12:13:30,818 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data1/current/BP-1283605086-172.17.0.2-1732277596080 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T12:13:30,819 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/cluster_7ea874c5-80ee-3af4-9029-3f879c851a58/data/data2/current/BP-1283605086-172.17.0.2-1732277596080 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T12:13:30,819 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T12:13:30,828 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3260f08c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T12:13:30,829 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@369f03e8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T12:13:30,829 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T12:13:30,829 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d5b118c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T12:13:30,829 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d9452bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76b8cba0-4103-a8ab-0ed3-3344711cc017/hadoop.log.dir/,STOPPED} 2024-11-22T12:13:30,839 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T12:13:30,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down